From 1c208e5946bcd27e1f367dd6cedc2ba68bb0770c Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Sun, 22 Mar 2020 12:14:43 -0500 Subject: [PATCH] docker-compose --- dockerfiles/migrations/docker-compose.yml | 37 ------------- dockerfiles/resync/Dockerfile | 54 ------------------- dockerfiles/resync/docker-compose.yml | 59 --------------------- dockerfiles/resync/startup_script.sh | 50 ------------------ dockerfiles/rinkeby/Dockerfile | 9 ---- dockerfiles/rinkeby/config.toml | 9 ---- dockerfiles/rinkeby/docker-compose.yml | 63 ----------------------- dockerfiles/super_node/docker-compose.yml | 47 +++++++++-------- dockerfiles/super_node/startup_script.sh | 33 +++++++++--- environments/superNodeBTC.toml | 2 +- environments/superNodeETH.toml | 2 +- 11 files changed, 53 insertions(+), 312 deletions(-) delete mode 100644 dockerfiles/migrations/docker-compose.yml delete mode 100644 dockerfiles/resync/Dockerfile delete mode 100644 dockerfiles/resync/docker-compose.yml delete mode 100755 dockerfiles/resync/startup_script.sh delete mode 100644 dockerfiles/rinkeby/Dockerfile delete mode 100644 dockerfiles/rinkeby/config.toml delete mode 100644 dockerfiles/rinkeby/docker-compose.yml diff --git a/dockerfiles/migrations/docker-compose.yml b/dockerfiles/migrations/docker-compose.yml deleted file mode 100644 index a4a8a646..00000000 --- a/dockerfiles/migrations/docker-compose.yml +++ /dev/null @@ -1,37 +0,0 @@ -version: '3.2' - -services: - db: - restart: always - image: postgres:10.12-alpine - environment: - POSTGRES_USER: "vdbm" - POSTGRES_DB: "vulcanize_public" - POSTGRES_PASSWORD: "password" - volumes: - - vulcanizedb_db_data:/var/lib/postgresql/data - expose: - - "5432" - ports: - - "127.0.0.1:8079:5432" - - migrations: - restart: on-failure - depends_on: - - db - build: - context: ./../../ - cache_from: - - alpine:latest - dockerfile: ./dockerfiles/migrations/Dockerfile - args: - USER: "vdbm" - environment: - DATABASE_NAME: "vulcanize_public" - DATABASE_HOSTNAME: "db" - DATABASE_PORT: 5432 - DATABASE_USER: "vdbm" - DATABASE_PASSWORD: "password" - -volumes: - vulcanizedb_db_data: \ No newline at end of file diff --git a/dockerfiles/resync/Dockerfile b/dockerfiles/resync/Dockerfile deleted file mode 100644 index 8b8995f5..00000000 --- a/dockerfiles/resync/Dockerfile +++ /dev/null @@ -1,54 +0,0 @@ -FROM golang:alpine - -RUN apk --update --no-cache add make git g++ linux-headers -# DEBUG -RUN apk add busybox-extras - -# this is probably a noob move, but I want apk from alpine for the above but need to avoid Go 1.13 below as this error still occurs https://github.com/ipfs/go-ipfs/issues/6603 -FROM golang:1.12.4 as builder - -# Get and build vulcanizedb -ADD . /go/src/github.com/vulcanize/vulcanizedb -WORKDIR /go/src/github.com/vulcanize/vulcanizedb -RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o vulcanizedb . - -# Get and build vulcanize's go-ipfs fork -RUN go get -u -d github.com/ipfs/go-ipfs -WORKDIR /go/src/github.com/ipfs/go-ipfs -RUN git remote add vulcanize https://github.com/vulcanize/go-ipfs.git -RUN git fetch vulcanize -RUN git checkout -b pg_ipfs vulcanize/postgres_update -RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o ipfs ./cmd/ipfs - -# Build migration tool -RUN go get -u -d github.com/pressly/goose/cmd/goose -WORKDIR /go/src/github.com/pressly/goose/cmd/goose -RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -tags='no_mysql no_sqlite' -o goose . - -WORKDIR /go/src/github.com/vulcanize/vulcanizedb - -# app container -FROM alpine - -ARG USER -ARG CONFIG_FILE - -RUN adduser -Du 5000 $USER -WORKDIR /app -RUN chown $USER /app -USER $USER - -# chown first so dir is writable -# note: using $USER is merged, but not in the stable release yet -COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/vulcanizedb/$CONFIG_FILE config.toml -COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/vulcanizedb/dockerfiles/resync/startup_script.sh . - - -# keep binaries immutable -COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/vulcanizedb vulcanizedb -COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose goose -COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/db/migrations migrations/vulcanizedb -COPY --from=builder /go/src/github.com/ipfs/go-ipfs/ipfs ipfs -# XXX dir is already writeable RUN touch vulcanizedb.log - -CMD ["./startup_script.sh"] \ No newline at end of file diff --git a/dockerfiles/resync/docker-compose.yml b/dockerfiles/resync/docker-compose.yml deleted file mode 100644 index 598147a0..00000000 --- a/dockerfiles/resync/docker-compose.yml +++ /dev/null @@ -1,59 +0,0 @@ -version: '3.2' - -services: - db: - restart: always - image: postgres:10.12-alpine - environment: - POSTGRES_USER: "vdbm" - POSTGRES_DB: "vulcanize_public" - POSTGRES_PASSWORD: "password" - volumes: - - vulcanizedb_db_data:/var/lib/postgresql/data - expose: - - "5432" - ports: - - "127.0.0.1:8079:5432" - - migrations: - restart: on-failure - depends_on: - - db - build: - context: ./../../ - cache_from: - - alpine:latest - dockerfile: ./dockerfiles/migrations/Dockerfile - environment: - DATABASE_NAME: "vulcanize_public" - DATABASE_HOSTNAME: "db" - DATABASE_PORT: 5432 - DATABASE_USER: "vdbm" - DATABASE_PASSWORD: "password" - - resync: - depends_on: - - db - - migrations - build: - context: ./../../ - cache_from: - - alpine:latest - - golang:1.12.4 - dockerfile: ./dockerfiles/super_node/Dockerfile - args: - USER: "vdbm" - CONFIG_FILE: ./environments/superNodeBTC.toml - environment: - IPFS_INIT: "true" - DATABASE_NAME: "vulcanize_public" - DATABASE_HOSTNAME: "db" - DATABASE_PORT: 5432 - DATABASE_USER: "vdbm" - DATABASE_PASSWORD: "password" - ports: - - "127.0.0.1:8082:8082" - - "127.0.0.1:8083:8083" - -volumes: - vulcanizedb_db_data: \ No newline at end of file diff --git a/dockerfiles/resync/startup_script.sh b/dockerfiles/resync/startup_script.sh deleted file mode 100755 index 98ea6923..00000000 --- a/dockerfiles/resync/startup_script.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/sh -# Runs the db migrations and starts the super node services - -# Exit if the variable tests fail -set -e -set +x - -# Check the database variables are set -test $DATABASE_HOSTNAME -test $DATABASE_NAME -test $DATABASE_PORT -test $DATABASE_USER -test $DATABASE_PASSWORD -test $IPFS_INIT -set +e - -# Export our database variables so that the IPFS Postgres plugin can use them -export IPFS_PGHOST=$DATABASE_HOSTNAME -export IPFS_PGUSER=$DATABASE_USER -export IPFS_PGDATABASE=$DATABASE_NAME -export IPFS_PGPORT=$DATABASE_PORT -export IPFS_PGPASSWORD=$DATABASE_PASSWORD - -# If IPFS_INIT is true -if [[ "$IPFS_INIT" = true ]] ; then - # initialize PG-IPFS - echo "Initializing Postgres-IPFS profile" - ./ipfs init --profile=postgresds -else - echo "IPFS profile already initialized, skipping initialization" -fi - -# If IPFS initialization was successful -if [[ $? -eq 0 ]]; then - echo "Beginning the vulcanizedb super node process" - ./vulcanizedb resync --config=config.toml 2>&1 | tee -a vulcanizedb.log & -else - echo "Could not initialize IPFS." - exit 1 -fi - -# If Vulcanizedb startup was successful -if [ $? -eq 0 ]; then - echo "Resync successfully booted" -else - echo "Could not start vulcanizedb resync process. Is the config file correct?" - exit 1 -fi - -tail -f vulcanizedb.log \ No newline at end of file diff --git a/dockerfiles/rinkeby/Dockerfile b/dockerfiles/rinkeby/Dockerfile deleted file mode 100644 index 7f5d5a7b..00000000 --- a/dockerfiles/rinkeby/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM golang:1.10.3-alpine3.7 - -RUN apk add --no-cache make gcc musl-dev - -ADD . /go/src/github.com/vulcanize/vulcanizedb -WORKDIR /go/src/github.com/vulcanize/vulcanizedb -RUN go build -o /app main.go - -ENTRYPOINT ["/app"] diff --git a/dockerfiles/rinkeby/config.toml b/dockerfiles/rinkeby/config.toml deleted file mode 100644 index 7fa6796e..00000000 --- a/dockerfiles/rinkeby/config.toml +++ /dev/null @@ -1,9 +0,0 @@ -[database] -name = "vulcanizedb" -hostname = "postgres" -port = 5432 -user = "postgres" -password = "postgres" - -[client] -ipcPath = "/geth/geth.ipc" diff --git a/dockerfiles/rinkeby/docker-compose.yml b/dockerfiles/rinkeby/docker-compose.yml deleted file mode 100644 index 4c3bd3aa..00000000 --- a/dockerfiles/rinkeby/docker-compose.yml +++ /dev/null @@ -1,63 +0,0 @@ -version: '2.2' - -services: - - vulcanizedb: - build: - context: ./../../ - dockerfile: dockerfiles/rinkeby/Dockerfile - container_name: rinkeby_vulcanizedb - command: "sync --starting-block-number 0 --config /config.toml" - volumes: - - "./config.toml:/config.toml" - - "vulcanizedb_geth_data:/geth" - networks: - vulcanizedb_net: - - migrations: - image: migrate/migrate:v3.3.0 - container_name: rinkeby_vulcanizedb_migrations - depends_on: - postgres: - condition: service_healthy - command: -database postgresql://postgres:postgres@postgres:5432/vulcanizedb?sslmode=disable -path /migrations up - volumes: - - ./../../db/migrations:/migrations - networks: - vulcanizedb_net: - - postgres: - image: postgres:9.6.5-alpine - container_name: rinkeby_vulcanizedb_postgres - environment: - POSTGRES_USER: postgres - POSTGRES_DB: vulcanizedb - POSTGRES_PASSWORD: postgres - volumes: - - "vulcanizedb_db_data:/var/lib/postgresql/data" - networks: - vulcanizedb_net: - healthcheck: - test: ["CMD", "pg_isready"] - interval: 5s - timeout: 5s - retries: 30 - - geth: - image: ethereum/client-go:v1.8.11 - container_name: rinkeby_vulcanizedb_geth - cpus: 0.3 - hostname: eth - command: '--rinkeby --rpc --rpcaddr="0.0.0.0" --rpcvhosts="geth"' - volumes: - - "vulcanizedb_geth_data:/root/.ethereum/rinkeby" - networks: - vulcanizedb_net: - -volumes: - vulcanizedb_geth_data: - vulcanizedb_db_data: - -networks: - vulcanizedb_net: - driver: bridge diff --git a/dockerfiles/super_node/docker-compose.yml b/dockerfiles/super_node/docker-compose.yml index e83c888a..c1540ec6 100644 --- a/dockerfiles/super_node/docker-compose.yml +++ b/dockerfiles/super_node/docker-compose.yml @@ -15,28 +15,9 @@ services: ports: - "127.0.0.1:8079:5432" - migrations: - restart: on-failure - depends_on: - - db - build: - context: ./../../ - cache_from: - - alpine:latest - dockerfile: ./dockerfiles/migrations/Dockerfile - args: - USER: "vdbm" - environment: - DATABASE_NAME: "vulcanize_public" - DATABASE_HOSTNAME: "db" - DATABASE_PORT: 5432 - DATABASE_USER: "vdbm" - DATABASE_PASSWORD: "password" - btc: depends_on: - db - - migrations build: context: ./../../ cache_from: @@ -47,7 +28,9 @@ services: USER: "vdbm" CONFIG_FILE: ./environments/superNodeBTC.toml environment: + VDB_COMMAND: "superNode" IPFS_INIT: "true" + IPFS_PATH: "/root/.btc/.ipfs" DATABASE_NAME: "vulcanize_public" DATABASE_HOSTNAME: "db" DATABASE_PORT: 5432 @@ -60,7 +43,6 @@ services: eth: depends_on: - db - - migrations build: context: ./../../ cache_from: @@ -71,7 +53,9 @@ services: USER: "vdbm" CONFIG_FILE: ./environments/superNodeETH.toml environment: + VDB_COMMAND: "superNode" IPFS_INIT: "true" + IPFS_PATH: "/root/.eth/.ipfs" DATABASE_NAME: "vulcanize_public" DATABASE_HOSTNAME: "db" DATABASE_PORT: 5432 @@ -81,11 +65,32 @@ services: - "127.0.0.1:8080:8080" - "127.0.0.1:8081:8081" + resync: + depends_on: + - db + build: + context: ./../../ + cache_from: + - alpine:latest + - golang:1.12.4 + dockerfile: ./dockerfiles/super_node/Dockerfile + args: + USER: "vdbm" + CONFIG_FILE: ./environments/superNodeETH.toml + environment: + VDB_COMMAND: "resync" + IPFS_INIT: "true" + IPFS_PATH: "/root/.eth_resync/.ipfs" + DATABASE_NAME: "vulcanize_public" + DATABASE_HOSTNAME: "db" + DATABASE_PORT: 5432 + DATABASE_USER: "vdbm" + DATABASE_PASSWORD: "password" + graphql: restart: always depends_on: - db - - migrations build: context: ./../../ cache_from: diff --git a/dockerfiles/super_node/startup_script.sh b/dockerfiles/super_node/startup_script.sh index 219096ef..0dc7e9ab 100755 --- a/dockerfiles/super_node/startup_script.sh +++ b/dockerfiles/super_node/startup_script.sh @@ -12,6 +12,8 @@ test $DATABASE_PORT test $DATABASE_USER test $DATABASE_PASSWORD test $IPFS_INIT +test $IPFS_PATH +test $VDB_COMMAND set +e # Export our database variables so that the IPFS Postgres plugin can use them @@ -21,19 +23,34 @@ export IPFS_PGDATABASE=$DATABASE_NAME export IPFS_PGPORT=$DATABASE_PORT export IPFS_PGPASSWORD=$DATABASE_PASSWORD -# If IPFS_INIT is true -if [[ "$IPFS_INIT" = true ]] ; then - # initialize PG-IPFS - echo "Initializing Postgres-IPFS profile" - ./ipfs init --profile=postgresds +# Construct the connection string for postgres +VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable + +# Run the DB migrations +echo "Connecting with: $VDB_PG_CONNECT" +echo "Running database migrations" +./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up + + +# If the db migrations ran without err +if [[ $? -eq 0 ]]; then + # and IPFS_INIT is true + if [[ "$IPFS_INIT" = true ]] ; then + # initialize PG-IPFS + echo "Initializing Postgres-IPFS profile" + ./ipfs init --profile=postgresds + else + echo "IPFS profile already initialized, skipping initialization" + fi else - echo "IPFS profile already initialized, skipping initialization" + echo "Could not run migrations. Are the database details correct?" + exit 1 fi # If IPFS initialization was successful if [[ $? -eq 0 ]]; then echo "Beginning the vulcanizedb super node process" - ./vulcanizedb superNode --config=config.toml 2>&1 | tee -a vulcanizedb.log & + ./vulcanizedb ${VDB_COMMAND} --config=config.toml 2>&1 | tee -a vulcanizedb.log & else echo "Could not initialize IPFS." exit 1 @@ -47,4 +64,4 @@ else exit 1 fi -tail -f vulcanizedb.log +tail -f vulcanizedb.log \ No newline at end of file diff --git a/environments/superNodeBTC.toml b/environments/superNodeBTC.toml index 42582702..9fa27600 100644 --- a/environments/superNodeBTC.toml +++ b/environments/superNodeBTC.toml @@ -6,7 +6,7 @@ password = "" # $DATABASE_PASSWORD [ipfs] - path = "/root/.ipfs" # $IPFS_PATH + path = "/root/.btc/.ipfs" # $IPFS_PATH [resync] chain = "bitcoin" # $RESYNC_CHAIN diff --git a/environments/superNodeETH.toml b/environments/superNodeETH.toml index 074da362..d4a55d58 100644 --- a/environments/superNodeETH.toml +++ b/environments/superNodeETH.toml @@ -6,7 +6,7 @@ password = "" # $DATABASE_PASSWORD [ipfs] - path = "/root/.ipfs" # $IPFS_PATH + path = "/root/.eth/.ipfs" # $IPFS_PATH [resync] chain = "ethereum" # $RESYNC_CHAIN