dockerfile for resync process

dockerfiles/resync/Dockerfile

@@ -0,0 +1,54 @@
+FROM golang:alpine
+
+RUN apk --update --no-cache add make git g++ linux-headers
+# DEBUG
+RUN apk add busybox-extras
+
+# this is probably a noob move, but I want apk from alpine for the above but need to avoid Go 1.13 below as this error still occurs https://github.com/ipfs/go-ipfs/issues/6603
+FROM golang:1.12.4 as builder
+
+# Get and build vulcanizedb
+ADD . /go/src/github.com/vulcanize/vulcanizedb
+WORKDIR /go/src/github.com/vulcanize/vulcanizedb
+RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o vulcanizedb .
+
+# Get and build vulcanize's go-ipfs fork
+RUN go get -u -d github.com/ipfs/go-ipfs
+WORKDIR /go/src/github.com/ipfs/go-ipfs
+RUN git remote add vulcanize https://github.com/vulcanize/go-ipfs.git
+RUN git fetch vulcanize
+RUN git checkout -b pg_ipfs vulcanize/postgres_update
+RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o ipfs ./cmd/ipfs
+
+# Build migration tool
+RUN go get -u -d github.com/pressly/goose/cmd/goose
+WORKDIR /go/src/github.com/pressly/goose/cmd/goose
+RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -tags='no_mysql no_sqlite' -o goose .
+
+WORKDIR /go/src/github.com/vulcanize/vulcanizedb
+
+# app container
+FROM alpine
+
+ARG USER
+ARG CONFIG_FILE
+
+RUN adduser -Du 5000 $USER
+WORKDIR /app
+RUN chown $USER /app
+USER $USER
+
+# chown first so dir is writable
+# note: using $USER is merged, but not in the stable release yet
+COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/vulcanizedb/$CONFIG_FILE config.toml
+COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/vulcanizedb/dockerfiles/resync/startup_script.sh .
+
+
+# keep binaries immutable
+COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/vulcanizedb vulcanizedb
+COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose goose
+COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/db/migrations migrations/vulcanizedb
+COPY --from=builder /go/src/github.com/ipfs/go-ipfs/ipfs ipfs
+# XXX dir is already writeable RUN touch vulcanizedb.log
+
+CMD ["./startup_script.sh"]

dockerfiles/resync/startup_script.sh

@@ -0,0 +1,65 @@
+#!/bin/sh
+# Runs the db migrations and starts the super node services
+
+# Exit if the variable tests fail
+set -e
+set +x
+
+# Check the database variables are set
+test $DATABASE_HOSTNAME
+test $DATABASE_NAME
+test $DATABASE_PORT
+test $DATABASE_USER
+test $DATABASE_PASSWORD
+test $IPFS_INIT
+set +e
+
+# Export our database variables so that the IPFS Postgres plugin can use them
+export IPFS_PGHOST=$DATABASE_HOSTNAME
+export IPFS_PGUSER=$DATABASE_USER
+export IPFS_PGDATABASE=$DATABASE_NAME
+export IPFS_PGPORT=$DATABASE_PORT
+export IPFS_PGPASSWORD=$DATABASE_PASSWORD
+
+# Construct the connection string for postgres
+VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable
+
+# Run the DB migrations
+echo "Connecting with: $VDB_PG_CONNECT"
+echo "Running database migrations"
+./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up
+
+
+# If the db migrations ran without err
+if [[ $? -eq 0 ]]; then
+    # and IPFS_INIT is true
+    if [[ "$IPFS_INIT" = true ]] ; then
+        # initialize PG-IPFS
+        echo "Initializing Postgres-IPFS profile"
+        ./ipfs init --profile=postgresds
+    else
+        echo "IPFS profile already initialized, skipping initialization"
+    fi
+else
+    echo "Could not run migrations. Are the database details correct?"
+    exit
+fi
+
+# If IPFS initialization was successful
+if [[ $? -eq 0 ]]; then
+    echo "Beginning the vulcanizedb super node process"
+    ./vulcanizedb resync --config=config.toml 2>&1 | tee -a vulcanizedb.log &
+else
+    echo "Could not initialize IPFS."
+    exit
+fi
+
+# If Vulcanizedb startup was successful
+if [ $? -eq 0 ]; then
+    echo "Resync successfully booted"
+else
+    echo "Could not start vulcanizedb resync process. Is the config file correct?"
+    exit
+fi
+
+tail -f vulcanizedb.log

dockerfiles/super_node/docker-compose.yml

@@ -28,11 +28,11 @@ services:
         CONFIG_FILE: ./environments/superNodeBTC.toml
     environment:
       IPFS_INIT: "true"
-      VDB_PG_NAME: "vulcanize_public"
-      VDB_PG_HOSTNAME: "db"
-      VDB_PG_PORT: 5432
-      VDB_PG_USER: "postgres"
-      VDB_PG_PASSWORD: "password"
+      DATABASE_NAME: "vulcanize_public"
+      DATABASE_HOSTNAME: "db"
+      DATABASE_PORT: 5432
+      DATABASE_USER: "postgres"
+      DATABASE_PASSWORD: "password"
     ports:
     - "127.0.0.1:8082:8082"
     - "127.0.0.1:8083:8083"
@@ -50,11 +50,11 @@ services:
         CONFIG_FILE: ./environments/superNodeETH.toml
     environment:
       IPFS_INIT: "true"
-      VDB_PG_NAME: "vulcanize_public"
-      VDB_PG_HOSTNAME: "db"
-      VDB_PG_PORT: 5432
-      VDB_PG_USER: "postgres"
-      VDB_PG_PASSWORD: "password"
+      DATABASE_NAME: "vulcanize_public"
+      DATABASE_HOSTNAME: "db"
+      DATABASE_PORT: 5432
+      DATABASE_USER: "postgres"
+      DATABASE_PASSWORD: "password"
     ports:
     - "127.0.0.1:8080:8080"
     - "127.0.0.1:8081:8081"
@@ -68,8 +68,6 @@ services:
       cache_from:
       - node:alpine
       dockerfile: ./dockerfiles/postgraphile/Dockerfile
-    environment:
-      DATABASE_URL: postgres://postgres:password@db:5432/vulcanize_public
     expose:
       - "5000"
     ports:
@@ -77,10 +75,10 @@ services:
     command: ["--plugins", "@graphile/pg-pubsub",
               "--subscriptions",
               "--simple-subscriptions",
-              "--connection", $DATABASE_URL,
+              "--connection", "postgres://postgres:password@db:5432/vulcanize_public",
               "--port", "5000",
-              "-n", "0.0.0.0"
-              "--schema", "public,btc,eth"
+              "-n", "0.0.0.0",
+              "--schema", "public,btc,eth",
               "--append-plugins", "postgraphile-plugin-connection-filter"]
 
 volumes:

dockerfiles/super_node/startup_script.sh

@@ -56,9 +56,9 @@ fi
 
 # If Vulcanizedb startup was successful
 if [ $? -eq 0 ]; then
-    echo "Seed node successfully booted"
+    echo "Super node successfully booted"
 else
-    echo "Could not start vulcanizedb syncPublishScreenAndServe process. Is the config file correct?"
+    echo "Could not start vulcanizedb super node process. Is the config file correct?"
     exit
 fi