From 1d4b37aca94b66bc9771ea67c1747b522d0c9caf Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Wed, 18 Mar 2020 12:21:29 -0500 Subject: [PATCH] split running migrations into separate compose step; docker-compose files for different processes --- cmd/resync.go | 4 +- dockerfiles/migrations/Dockerfile | 40 ++++++++++++++ dockerfiles/migrations/docker-compose.yml | 37 +++++++++++++ dockerfiles/migrations/startup_script.sh | 32 +++++++++++ dockerfiles/postgraphile/docker-compose.yml | 60 +++++++++++++++++++++ dockerfiles/resync/docker-compose.yml | 59 ++++++++++++++++++++ dockerfiles/resync/startup_script.sh | 31 +++-------- dockerfiles/super_node/docker-compose.yml | 59 +++++++++++++------- dockerfiles/super_node/startup_script.sh | 31 +++-------- 9 files changed, 286 insertions(+), 67 deletions(-) create mode 100644 dockerfiles/migrations/Dockerfile create mode 100644 dockerfiles/migrations/docker-compose.yml create mode 100755 dockerfiles/migrations/startup_script.sh create mode 100644 dockerfiles/postgraphile/docker-compose.yml create mode 100644 dockerfiles/resync/docker-compose.yml mode change 100644 => 100755 dockerfiles/resync/startup_script.sh diff --git a/cmd/resync.go b/cmd/resync.go index 121000d2..fd48cfca 100644 --- a/cmd/resync.go +++ b/cmd/resync.go @@ -16,8 +16,6 @@ package cmd import ( - "fmt" - log "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -56,5 +54,5 @@ func rsyncCmdCommand() { if err := rService.Resync(); err != nil { logWithCommand.Fatal(err) } - fmt.Printf("%s %s resync finished", rConfig.Chain.String(), rConfig.ResyncType.String()) + logWithCommand.Infof("%s %s resync finished", rConfig.Chain.String(), rConfig.ResyncType.String()) } diff --git a/dockerfiles/migrations/Dockerfile b/dockerfiles/migrations/Dockerfile new file mode 100644 index 00000000..fd4c9139 --- /dev/null +++ b/dockerfiles/migrations/Dockerfile @@ -0,0 +1,40 @@ +FROM golang:alpine + +RUN apk --update --no-cache add make git g++ linux-headers +# DEBUG +RUN apk add busybox-extras + +# this is probably a noob move, but I want apk from alpine for the above but need to avoid Go 1.13 below as this error still occurs https://github.com/ipfs/go-ipfs/issues/6603 +FROM golang:1.12.4 as builder + +# Get and build vulcanizedb +ADD . /go/src/github.com/vulcanize/vulcanizedb + +# Build migration tool +RUN go get -u -d github.com/pressly/goose/cmd/goose +WORKDIR /go/src/github.com/pressly/goose/cmd/goose +RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -tags='no_mysql no_sqlite' -o goose . + +WORKDIR /go/src/github.com/vulcanize/vulcanizedb + +# app container +FROM alpine + +ARG USER + +RUN adduser -Du 5000 $USER +WORKDIR /app +RUN chown $USER /app +USER $USER + +# chown first so dir is writable +# note: using $USER is merged, but not in the stable release yet +COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/vulcanizedb/dockerfiles/migrations/startup_script.sh . + + +# keep binaries immutable +COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose goose +COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/db/migrations migrations/vulcanizedb +# XXX dir is already writeable RUN touch vulcanizedb.log + +CMD ["./startup_script.sh"] \ No newline at end of file diff --git a/dockerfiles/migrations/docker-compose.yml b/dockerfiles/migrations/docker-compose.yml new file mode 100644 index 00000000..a4a8a646 --- /dev/null +++ b/dockerfiles/migrations/docker-compose.yml @@ -0,0 +1,37 @@ +version: '3.2' + +services: + db: + restart: always + image: postgres:10.12-alpine + environment: + POSTGRES_USER: "vdbm" + POSTGRES_DB: "vulcanize_public" + POSTGRES_PASSWORD: "password" + volumes: + - vulcanizedb_db_data:/var/lib/postgresql/data + expose: + - "5432" + ports: + - "127.0.0.1:8079:5432" + + migrations: + restart: on-failure + depends_on: + - db + build: + context: ./../../ + cache_from: + - alpine:latest + dockerfile: ./dockerfiles/migrations/Dockerfile + args: + USER: "vdbm" + environment: + DATABASE_NAME: "vulcanize_public" + DATABASE_HOSTNAME: "db" + DATABASE_PORT: 5432 + DATABASE_USER: "vdbm" + DATABASE_PASSWORD: "password" + +volumes: + vulcanizedb_db_data: \ No newline at end of file diff --git a/dockerfiles/migrations/startup_script.sh b/dockerfiles/migrations/startup_script.sh new file mode 100755 index 00000000..ca5c5159 --- /dev/null +++ b/dockerfiles/migrations/startup_script.sh @@ -0,0 +1,32 @@ +#!/bin/sh +# Runs the db migrations and starts the super node services + +# Exit if the variable tests fail +set -e +set +x + +# Check the database variables are set +test $DATABASE_HOSTNAME +test $DATABASE_NAME +test $DATABASE_PORT +test $DATABASE_USER +test $DATABASE_PASSWORD +set +e + +# Construct the connection string for postgres +VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable + +# Run the DB migrations +echo "Connecting with: $VDB_PG_CONNECT" +echo "Running database migrations" +./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up + + +# If the db migrations ran without err +if [[ $? -eq 0 ]]; then + echo "Migrations ran successfully" + exit 0 +else + echo "Could not run migrations. Are the database details correct?" + exit 1 +fi \ No newline at end of file diff --git a/dockerfiles/postgraphile/docker-compose.yml b/dockerfiles/postgraphile/docker-compose.yml new file mode 100644 index 00000000..5ee8059e --- /dev/null +++ b/dockerfiles/postgraphile/docker-compose.yml @@ -0,0 +1,60 @@ +version: '3.2' + +services: + db: + restart: always + image: postgres:10.12-alpine + environment: + POSTGRES_USER: "vdbm" + POSTGRES_DB: "vulcanize_public" + POSTGRES_PASSWORD: "password" + volumes: + - vulcanizedb_db_data:/var/lib/postgresql/data + expose: + - "5432" + ports: + - "127.0.0.1:8079:5432" + + migrations: + restart: on-failure + depends_on: + - db + build: + context: ./../../ + cache_from: + - alpine:latest + dockerfile: ./dockerfiles/migrations/Dockerfile + args: + USER: "vdbm" + environment: + DATABASE_NAME: "vulcanize_public" + DATABASE_HOSTNAME: "db" + DATABASE_PORT: 5432 + DATABASE_USER: "vdbm" + DATABASE_PASSWORD: "password" + + graphql: + restart: always + depends_on: + - db + - migrations + build: + context: ./../../ + cache_from: + - node:alpine + dockerfile: ./dockerfiles/postgraphile/Dockerfile + expose: + - "5000" + ports: + - "127.0.0.1:5000:5000" + command: ["--plugins", "@graphile/pg-pubsub", + "--subscriptions", + "--simple-subscriptions", + "--connection", "postgres://vdbm:password@db:5432/vulcanize_public", + "--port", "5000", + "-n", "0.0.0.0", + "--schema", "public,btc,eth", + "--append-plugins", "postgraphile-plugin-connection-filter"] + +volumes: + vulcanizedb_db_data: \ No newline at end of file diff --git a/dockerfiles/resync/docker-compose.yml b/dockerfiles/resync/docker-compose.yml new file mode 100644 index 00000000..598147a0 --- /dev/null +++ b/dockerfiles/resync/docker-compose.yml @@ -0,0 +1,59 @@ +version: '3.2' + +services: + db: + restart: always + image: postgres:10.12-alpine + environment: + POSTGRES_USER: "vdbm" + POSTGRES_DB: "vulcanize_public" + POSTGRES_PASSWORD: "password" + volumes: + - vulcanizedb_db_data:/var/lib/postgresql/data + expose: + - "5432" + ports: + - "127.0.0.1:8079:5432" + + migrations: + restart: on-failure + depends_on: + - db + build: + context: ./../../ + cache_from: + - alpine:latest + dockerfile: ./dockerfiles/migrations/Dockerfile + environment: + DATABASE_NAME: "vulcanize_public" + DATABASE_HOSTNAME: "db" + DATABASE_PORT: 5432 + DATABASE_USER: "vdbm" + DATABASE_PASSWORD: "password" + + resync: + depends_on: + - db + - migrations + build: + context: ./../../ + cache_from: + - alpine:latest + - golang:1.12.4 + dockerfile: ./dockerfiles/super_node/Dockerfile + args: + USER: "vdbm" + CONFIG_FILE: ./environments/superNodeBTC.toml + environment: + IPFS_INIT: "true" + DATABASE_NAME: "vulcanize_public" + DATABASE_HOSTNAME: "db" + DATABASE_PORT: 5432 + DATABASE_USER: "vdbm" + DATABASE_PASSWORD: "password" + ports: + - "127.0.0.1:8082:8082" + - "127.0.0.1:8083:8083" + +volumes: + vulcanizedb_db_data: \ No newline at end of file diff --git a/dockerfiles/resync/startup_script.sh b/dockerfiles/resync/startup_script.sh old mode 100644 new mode 100755 index ddeec23f..98ea6923 --- a/dockerfiles/resync/startup_script.sh +++ b/dockerfiles/resync/startup_script.sh @@ -21,28 +21,13 @@ export IPFS_PGDATABASE=$DATABASE_NAME export IPFS_PGPORT=$DATABASE_PORT export IPFS_PGPASSWORD=$DATABASE_PASSWORD -# Construct the connection string for postgres -VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable - -# Run the DB migrations -echo "Connecting with: $VDB_PG_CONNECT" -echo "Running database migrations" -./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up - - -# If the db migrations ran without err -if [[ $? -eq 0 ]]; then - # and IPFS_INIT is true - if [[ "$IPFS_INIT" = true ]] ; then - # initialize PG-IPFS - echo "Initializing Postgres-IPFS profile" - ./ipfs init --profile=postgresds - else - echo "IPFS profile already initialized, skipping initialization" - fi +# If IPFS_INIT is true +if [[ "$IPFS_INIT" = true ]] ; then + # initialize PG-IPFS + echo "Initializing Postgres-IPFS profile" + ./ipfs init --profile=postgresds else - echo "Could not run migrations. Are the database details correct?" - exit + echo "IPFS profile already initialized, skipping initialization" fi # If IPFS initialization was successful @@ -51,7 +36,7 @@ if [[ $? -eq 0 ]]; then ./vulcanizedb resync --config=config.toml 2>&1 | tee -a vulcanizedb.log & else echo "Could not initialize IPFS." - exit + exit 1 fi # If Vulcanizedb startup was successful @@ -59,7 +44,7 @@ if [ $? -eq 0 ]; then echo "Resync successfully booted" else echo "Could not start vulcanizedb resync process. Is the config file correct?" - exit + exit 1 fi tail -f vulcanizedb.log \ No newline at end of file diff --git a/dockerfiles/super_node/docker-compose.yml b/dockerfiles/super_node/docker-compose.yml index f8995549..e83c888a 100644 --- a/dockerfiles/super_node/docker-compose.yml +++ b/dockerfiles/super_node/docker-compose.yml @@ -5,64 +5,87 @@ services: restart: always image: postgres:10.12-alpine environment: - POSTGRES_USER: "postgres" + POSTGRES_USER: "vdbm" POSTGRES_DB: "vulcanize_public" POSTGRES_PASSWORD: "password" volumes: - - vulcanizedb_db_data:/var/lib/postgresql/data + - vulcanizedb_db_data:/var/lib/postgresql/data expose: - - "5432" + - "5432" ports: - - "127.0.0.1:8079:5432" + - "127.0.0.1:8079:5432" - btc: + migrations: + restart: on-failure depends_on: - - db + - db build: context: ./../../ cache_from: - - alpine:latest - - golang:1.12.4 + - alpine:latest + dockerfile: ./dockerfiles/migrations/Dockerfile + args: + USER: "vdbm" + environment: + DATABASE_NAME: "vulcanize_public" + DATABASE_HOSTNAME: "db" + DATABASE_PORT: 5432 + DATABASE_USER: "vdbm" + DATABASE_PASSWORD: "password" + + btc: + depends_on: + - db + - migrations + build: + context: ./../../ + cache_from: + - alpine:latest + - golang:1.12.4 dockerfile: ./dockerfiles/super_node/Dockerfile args: + USER: "vdbm" CONFIG_FILE: ./environments/superNodeBTC.toml environment: IPFS_INIT: "true" DATABASE_NAME: "vulcanize_public" DATABASE_HOSTNAME: "db" DATABASE_PORT: 5432 - DATABASE_USER: "postgres" + DATABASE_USER: "vdbm" DATABASE_PASSWORD: "password" ports: - - "127.0.0.1:8082:8082" - - "127.0.0.1:8083:8083" + - "127.0.0.1:8082:8082" + - "127.0.0.1:8083:8083" eth: depends_on: - - db + - db + - migrations build: context: ./../../ cache_from: - - alpine:latest - - golang:1.12.4 + - alpine:latest + - golang:1.12.4 dockerfile: ./dockerfiles/super_node/Dockerfile args: + USER: "vdbm" CONFIG_FILE: ./environments/superNodeETH.toml environment: IPFS_INIT: "true" DATABASE_NAME: "vulcanize_public" DATABASE_HOSTNAME: "db" DATABASE_PORT: 5432 - DATABASE_USER: "postgres" + DATABASE_USER: "vdbm" DATABASE_PASSWORD: "password" ports: - - "127.0.0.1:8080:8080" - - "127.0.0.1:8081:8081" + - "127.0.0.1:8080:8080" + - "127.0.0.1:8081:8081" graphql: restart: always depends_on: - db + - migrations build: context: ./../../ cache_from: @@ -75,7 +98,7 @@ services: command: ["--plugins", "@graphile/pg-pubsub", "--subscriptions", "--simple-subscriptions", - "--connection", "postgres://postgres:password@db:5432/vulcanize_public", + "--connection", "postgres://vdbm:password@db:5432/vulcanize_public", "--port", "5000", "-n", "0.0.0.0", "--schema", "public,btc,eth", diff --git a/dockerfiles/super_node/startup_script.sh b/dockerfiles/super_node/startup_script.sh index 06c04a2e..219096ef 100755 --- a/dockerfiles/super_node/startup_script.sh +++ b/dockerfiles/super_node/startup_script.sh @@ -21,28 +21,13 @@ export IPFS_PGDATABASE=$DATABASE_NAME export IPFS_PGPORT=$DATABASE_PORT export IPFS_PGPASSWORD=$DATABASE_PASSWORD -# Construct the connection string for postgres -VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable - -# Run the DB migrations -echo "Connecting with: $VDB_PG_CONNECT" -echo "Running database migrations" -./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up - - -# If the db migrations ran without err -if [[ $? -eq 0 ]]; then - # and IPFS_INIT is true - if [[ "$IPFS_INIT" = true ]] ; then - # initialize PG-IPFS - echo "Initializing Postgres-IPFS profile" - ./ipfs init --profile=postgresds - else - echo "IPFS profile already initialized, skipping initialization" - fi +# If IPFS_INIT is true +if [[ "$IPFS_INIT" = true ]] ; then + # initialize PG-IPFS + echo "Initializing Postgres-IPFS profile" + ./ipfs init --profile=postgresds else - echo "Could not run migrations. Are the database details correct?" - exit + echo "IPFS profile already initialized, skipping initialization" fi # If IPFS initialization was successful @@ -51,7 +36,7 @@ if [[ $? -eq 0 ]]; then ./vulcanizedb superNode --config=config.toml 2>&1 | tee -a vulcanizedb.log & else echo "Could not initialize IPFS." - exit + exit 1 fi # If Vulcanizedb startup was successful @@ -59,7 +44,7 @@ if [ $? -eq 0 ]; then echo "Super node successfully booted" else echo "Could not start vulcanizedb super node process. Is the config file correct?" - exit + exit 1 fi tail -f vulcanizedb.log