split running migrations into separate compose step; docker-compose files for different processes

cmd/resync.go

@@ -16,8 +16,6 @@
 package cmd
 
 import (
-	"fmt"
-
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
@@ -56,5 +54,5 @@ func rsyncCmdCommand() {
 	if err := rService.Resync(); err != nil {
 		logWithCommand.Fatal(err)
 	}
-	fmt.Printf("%s %s resync finished", rConfig.Chain.String(), rConfig.ResyncType.String())
+	logWithCommand.Infof("%s %s resync finished", rConfig.Chain.String(), rConfig.ResyncType.String())
 }

dockerfiles/migrations/Dockerfile

@@ -0,0 +1,40 @@
+FROM golang:alpine
+
+RUN apk --update --no-cache add make git g++ linux-headers
+# DEBUG
+RUN apk add busybox-extras
+
+# this is probably a noob move, but I want apk from alpine for the above but need to avoid Go 1.13 below as this error still occurs https://github.com/ipfs/go-ipfs/issues/6603
+FROM golang:1.12.4 as builder
+
+# Get and build vulcanizedb
+ADD . /go/src/github.com/vulcanize/vulcanizedb
+
+# Build migration tool
+RUN go get -u -d github.com/pressly/goose/cmd/goose
+WORKDIR /go/src/github.com/pressly/goose/cmd/goose
+RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -tags='no_mysql no_sqlite' -o goose .
+
+WORKDIR /go/src/github.com/vulcanize/vulcanizedb
+
+# app container
+FROM alpine
+
+ARG USER
+
+RUN adduser -Du 5000 $USER
+WORKDIR /app
+RUN chown $USER /app
+USER $USER
+
+# chown first so dir is writable
+# note: using $USER is merged, but not in the stable release yet
+COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/vulcanizedb/dockerfiles/migrations/startup_script.sh .
+
+
+# keep binaries immutable
+COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose goose
+COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/db/migrations migrations/vulcanizedb
+# XXX dir is already writeable RUN touch vulcanizedb.log
+
+CMD ["./startup_script.sh"]

dockerfiles/migrations/docker-compose.yml

@@ -0,0 +1,37 @@
+version: '3.2'
+
+services:
+  db:
+    restart: always
+    image: postgres:10.12-alpine
+    environment:
+      POSTGRES_USER: "vdbm"
+      POSTGRES_DB: "vulcanize_public"
+      POSTGRES_PASSWORD: "password"
+    volumes:
+     - vulcanizedb_db_data:/var/lib/postgresql/data
+    expose:
+     - "5432"
+    ports:
+     - "127.0.0.1:8079:5432"
+
+  migrations:
+    restart: on-failure
+    depends_on:
+      - db
+    build:
+      context: ./../../
+      cache_from:
+       - alpine:latest
+      dockerfile: ./dockerfiles/migrations/Dockerfile
+      args:
+        USER: "vdbm"
+    environment:
+      DATABASE_NAME: "vulcanize_public"
+      DATABASE_HOSTNAME: "db"
+      DATABASE_PORT: 5432
+      DATABASE_USER: "vdbm"
+      DATABASE_PASSWORD: "password"
+
+volumes:
+  vulcanizedb_db_data:

dockerfiles/migrations/startup_script.sh

@@ -0,0 +1,32 @@
+#!/bin/sh
+# Runs the db migrations and starts the super node services
+
+# Exit if the variable tests fail
+set -e
+set +x
+
+# Check the database variables are set
+test $DATABASE_HOSTNAME
+test $DATABASE_NAME
+test $DATABASE_PORT
+test $DATABASE_USER
+test $DATABASE_PASSWORD
+set +e
+
+# Construct the connection string for postgres
+VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable
+
+# Run the DB migrations
+echo "Connecting with: $VDB_PG_CONNECT"
+echo "Running database migrations"
+./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up
+
+
+# If the db migrations ran without err
+if [[ $? -eq 0 ]]; then
+    echo "Migrations ran successfully"
+    exit 0
+else
+    echo "Could not run migrations. Are the database details correct?"
+    exit 1
+fi

dockerfiles/postgraphile/docker-compose.yml

@@ -0,0 +1,60 @@
+version: '3.2'
+
+services:
+  db:
+    restart: always
+    image: postgres:10.12-alpine
+    environment:
+      POSTGRES_USER: "vdbm"
+      POSTGRES_DB: "vulcanize_public"
+      POSTGRES_PASSWORD: "password"
+    volumes:
+     - vulcanizedb_db_data:/var/lib/postgresql/data
+    expose:
+     - "5432"
+    ports:
+     - "127.0.0.1:8079:5432"
+
+  migrations:
+    restart: on-failure
+    depends_on:
+     - db
+    build:
+      context: ./../../
+      cache_from:
+        - alpine:latest
+      dockerfile: ./dockerfiles/migrations/Dockerfile
+      args:
+        USER: "vdbm"
+    environment:
+      DATABASE_NAME: "vulcanize_public"
+      DATABASE_HOSTNAME: "db"
+      DATABASE_PORT: 5432
+      DATABASE_USER: "vdbm"
+      DATABASE_PASSWORD: "password"
+
+  graphql:
+    restart: always
+    depends_on:
+     - db
+     - migrations
+    build:
+      context: ./../../
+      cache_from:
+       - node:alpine
+      dockerfile: ./dockerfiles/postgraphile/Dockerfile
+    expose:
+     - "5000"
+    ports:
+     - "127.0.0.1:5000:5000"
+    command: ["--plugins", "@graphile/pg-pubsub",
+              "--subscriptions",
+              "--simple-subscriptions",
+              "--connection", "postgres://vdbm:password@db:5432/vulcanize_public",
+              "--port", "5000",
+              "-n", "0.0.0.0",
+              "--schema", "public,btc,eth",
+              "--append-plugins", "postgraphile-plugin-connection-filter"]
+
+volumes:
+  vulcanizedb_db_data:

dockerfiles/resync/docker-compose.yml

@@ -0,0 +1,59 @@
+version: '3.2'
+
+services:
+  db:
+    restart: always
+    image: postgres:10.12-alpine
+    environment:
+      POSTGRES_USER: "vdbm"
+      POSTGRES_DB: "vulcanize_public"
+      POSTGRES_PASSWORD: "password"
+    volumes:
+     - vulcanizedb_db_data:/var/lib/postgresql/data
+    expose:
+     - "5432"
+    ports:
+     - "127.0.0.1:8079:5432"
+
+  migrations:
+    restart: on-failure
+    depends_on:
+     - db
+    build:
+      context: ./../../
+      cache_from:
+        - alpine:latest
+      dockerfile: ./dockerfiles/migrations/Dockerfile
+    environment:
+      DATABASE_NAME: "vulcanize_public"
+      DATABASE_HOSTNAME: "db"
+      DATABASE_PORT: 5432
+      DATABASE_USER: "vdbm"
+      DATABASE_PASSWORD: "password"
+
+  resync:
+    depends_on:
+     - db
+     - migrations
+    build:
+      context: ./../../
+      cache_from:
+        - alpine:latest
+        - golang:1.12.4
+      dockerfile: ./dockerfiles/super_node/Dockerfile
+      args:
+        USER: "vdbm"
+        CONFIG_FILE: ./environments/superNodeBTC.toml
+    environment:
+      IPFS_INIT: "true"
+      DATABASE_NAME: "vulcanize_public"
+      DATABASE_HOSTNAME: "db"
+      DATABASE_PORT: 5432
+      DATABASE_USER: "vdbm"
+      DATABASE_PASSWORD: "password"
+    ports:
+     - "127.0.0.1:8082:8082"
+     - "127.0.0.1:8083:8083"
+
+volumes:
+  vulcanizedb_db_data:

dockerfiles/resync/startup_script.sh

@@ -21,28 +21,13 @@ export IPFS_PGDATABASE=$DATABASE_NAME
 export IPFS_PGPORT=$DATABASE_PORT
 export IPFS_PGPASSWORD=$DATABASE_PASSWORD
 
-# Construct the connection string for postgres
+# If IPFS_INIT is true
-VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable
+if [[ "$IPFS_INIT" = true ]] ; then
-
+    # initialize PG-IPFS
-# Run the DB migrations
+    echo "Initializing Postgres-IPFS profile"
-echo "Connecting with: $VDB_PG_CONNECT"
+    ./ipfs init --profile=postgresds
-echo "Running database migrations"
-./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up
-
-
-# If the db migrations ran without err
-if [[ $? -eq 0 ]]; then
-    # and IPFS_INIT is true
-    if [[ "$IPFS_INIT" = true ]] ; then
-        # initialize PG-IPFS
-        echo "Initializing Postgres-IPFS profile"
-        ./ipfs init --profile=postgresds
-    else
-        echo "IPFS profile already initialized, skipping initialization"
-    fi
 else
-    echo "Could not run migrations. Are the database details correct?"
+    echo "IPFS profile already initialized, skipping initialization"
-    exit
 fi
 
 # If IPFS initialization was successful
@@ -51,7 +36,7 @@ if [[ $? -eq 0 ]]; then
     ./vulcanizedb resync --config=config.toml 2>&1 | tee -a vulcanizedb.log &
 else
     echo "Could not initialize IPFS."
-    exit
+    exit 1
 fi
 
 # If Vulcanizedb startup was successful
@@ -59,7 +44,7 @@ if [ $? -eq 0 ]; then
     echo "Resync successfully booted"
 else
     echo "Could not start vulcanizedb resync process. Is the config file correct?"
-    exit
+    exit 1
 fi
 
 tail -f vulcanizedb.log

dockerfiles/super_node/docker-compose.yml

@@ -5,64 +5,87 @@ services:
     restart: always
     image: postgres:10.12-alpine
     environment:
-      POSTGRES_USER: "postgres"
+      POSTGRES_USER: "vdbm"
       POSTGRES_DB: "vulcanize_public"
       POSTGRES_PASSWORD: "password"
     volumes:
-    - vulcanizedb_db_data:/var/lib/postgresql/data
+     - vulcanizedb_db_data:/var/lib/postgresql/data
     expose:
-    - "5432"
+     - "5432"
     ports:
-    - "127.0.0.1:8079:5432"
+     - "127.0.0.1:8079:5432"
 
-  btc:
+  migrations:
+    restart: on-failure
     depends_on:
-     - db
+      - db
     build:
       context: ./../../
       cache_from:
-        - alpine:latest
+       - alpine:latest
-        - golang:1.12.4
+      dockerfile: ./dockerfiles/migrations/Dockerfile
+      args:
+        USER: "vdbm"
+    environment:
+      DATABASE_NAME: "vulcanize_public"
+      DATABASE_HOSTNAME: "db"
+      DATABASE_PORT: 5432
+      DATABASE_USER: "vdbm"
+      DATABASE_PASSWORD: "password"
+
+  btc:
+    depends_on:
+      - db
+      - migrations
+    build:
+      context: ./../../
+      cache_from:
+       - alpine:latest
+       - golang:1.12.4
       dockerfile: ./dockerfiles/super_node/Dockerfile
       args:
+        USER: "vdbm"
         CONFIG_FILE: ./environments/superNodeBTC.toml
     environment:
       IPFS_INIT: "true"
       DATABASE_NAME: "vulcanize_public"
       DATABASE_HOSTNAME: "db"
       DATABASE_PORT: 5432
-      DATABASE_USER: "postgres"
+      DATABASE_USER: "vdbm"
       DATABASE_PASSWORD: "password"
     ports:
-    - "127.0.0.1:8082:8082"
+     - "127.0.0.1:8082:8082"
-    - "127.0.0.1:8083:8083"
+     - "127.0.0.1:8083:8083"
 
   eth:
     depends_on:
-    - db
+      - db
+      - migrations
     build:
       context: ./../../
       cache_from:
-      - alpine:latest
+       - alpine:latest
-      - golang:1.12.4
+       - golang:1.12.4
       dockerfile: ./dockerfiles/super_node/Dockerfile
       args:
+        USER: "vdbm"
         CONFIG_FILE: ./environments/superNodeETH.toml
     environment:
       IPFS_INIT: "true"
       DATABASE_NAME: "vulcanize_public"
       DATABASE_HOSTNAME: "db"
       DATABASE_PORT: 5432
-      DATABASE_USER: "postgres"
+      DATABASE_USER: "vdbm"
       DATABASE_PASSWORD: "password"
     ports:
-    - "127.0.0.1:8080:8080"
+     - "127.0.0.1:8080:8080"
-    - "127.0.0.1:8081:8081"
+     - "127.0.0.1:8081:8081"
 
   graphql:
     restart: always
     depends_on:
       - db
+      - migrations
     build:
       context: ./../../
       cache_from:
@@ -75,7 +98,7 @@ services:
     command: ["--plugins", "@graphile/pg-pubsub",
               "--subscriptions",
               "--simple-subscriptions",
-              "--connection", "postgres://postgres:password@db:5432/vulcanize_public",
+              "--connection", "postgres://vdbm:password@db:5432/vulcanize_public",
               "--port", "5000",
               "-n", "0.0.0.0",
               "--schema", "public,btc,eth",

dockerfiles/super_node/startup_script.sh

@@ -21,28 +21,13 @@ export IPFS_PGDATABASE=$DATABASE_NAME
 export IPFS_PGPORT=$DATABASE_PORT
 export IPFS_PGPASSWORD=$DATABASE_PASSWORD
 
-# Construct the connection string for postgres
+# If IPFS_INIT is true
-VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable
+if [[ "$IPFS_INIT" = true ]] ; then
-
+    # initialize PG-IPFS
-# Run the DB migrations
+    echo "Initializing Postgres-IPFS profile"
-echo "Connecting with: $VDB_PG_CONNECT"
+    ./ipfs init --profile=postgresds
-echo "Running database migrations"
-./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up
-
-
-# If the db migrations ran without err
-if [[ $? -eq 0 ]]; then
-    # and IPFS_INIT is true
-    if [[ "$IPFS_INIT" = true ]] ; then
-        # initialize PG-IPFS
-        echo "Initializing Postgres-IPFS profile"
-        ./ipfs init --profile=postgresds
-    else
-        echo "IPFS profile already initialized, skipping initialization"
-    fi
 else
-    echo "Could not run migrations. Are the database details correct?"
+    echo "IPFS profile already initialized, skipping initialization"
-    exit
 fi
 
 # If IPFS initialization was successful
@@ -51,7 +36,7 @@ if [[ $? -eq 0 ]]; then
     ./vulcanizedb superNode --config=config.toml 2>&1 | tee -a vulcanizedb.log &
 else
     echo "Could not initialize IPFS."
-    exit
+    exit 1
 fi
 
 # If Vulcanizedb startup was successful
@@ -59,7 +44,7 @@ if [ $? -eq 0 ]; then
     echo "Super node successfully booted"
 else
     echo "Could not start vulcanizedb super node process. Is the config file correct?"
-    exit
+    exit 1
 fi
 
 tail -f vulcanizedb.log