docker-compose

dockerfiles/migrations/docker-compose.yml

@@ -1,37 +0,0 @@
-version: '3.2'
-
-services:
-  db:
-    restart: always
-    image: postgres:10.12-alpine
-    environment:
-      POSTGRES_USER: "vdbm"
-      POSTGRES_DB: "vulcanize_public"
-      POSTGRES_PASSWORD: "password"
-    volumes:
-     - vulcanizedb_db_data:/var/lib/postgresql/data
-    expose:
-     - "5432"
-    ports:
-     - "127.0.0.1:8079:5432"
-
-  migrations:
-    restart: on-failure
-    depends_on:
-      - db
-    build:
-      context: ./../../
-      cache_from:
-       - alpine:latest
-      dockerfile: ./dockerfiles/migrations/Dockerfile
-      args:
-        USER: "vdbm"
-    environment:
-      DATABASE_NAME: "vulcanize_public"
-      DATABASE_HOSTNAME: "db"
-      DATABASE_PORT: 5432
-      DATABASE_USER: "vdbm"
-      DATABASE_PASSWORD: "password"
-
-volumes:
-  vulcanizedb_db_data:

dockerfiles/resync/Dockerfile

@@ -1,54 +0,0 @@
-FROM golang:alpine
-
-RUN apk --update --no-cache add make git g++ linux-headers
-# DEBUG
-RUN apk add busybox-extras
-
-# this is probably a noob move, but I want apk from alpine for the above but need to avoid Go 1.13 below as this error still occurs https://github.com/ipfs/go-ipfs/issues/6603
-FROM golang:1.12.4 as builder
-
-# Get and build vulcanizedb
-ADD . /go/src/github.com/vulcanize/vulcanizedb
-WORKDIR /go/src/github.com/vulcanize/vulcanizedb
-RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o vulcanizedb .
-
-# Get and build vulcanize's go-ipfs fork
-RUN go get -u -d github.com/ipfs/go-ipfs
-WORKDIR /go/src/github.com/ipfs/go-ipfs
-RUN git remote add vulcanize https://github.com/vulcanize/go-ipfs.git
-RUN git fetch vulcanize
-RUN git checkout -b pg_ipfs vulcanize/postgres_update
-RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o ipfs ./cmd/ipfs
-
-# Build migration tool
-RUN go get -u -d github.com/pressly/goose/cmd/goose
-WORKDIR /go/src/github.com/pressly/goose/cmd/goose
-RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -tags='no_mysql no_sqlite' -o goose .
-
-WORKDIR /go/src/github.com/vulcanize/vulcanizedb
-
-# app container
-FROM alpine
-
-ARG USER
-ARG CONFIG_FILE
-
-RUN adduser -Du 5000 $USER
-WORKDIR /app
-RUN chown $USER /app
-USER $USER
-
-# chown first so dir is writable
-# note: using $USER is merged, but not in the stable release yet
-COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/vulcanizedb/$CONFIG_FILE config.toml
-COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/vulcanizedb/dockerfiles/resync/startup_script.sh .
-
-
-# keep binaries immutable
-COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/vulcanizedb vulcanizedb
-COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose goose
-COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/db/migrations migrations/vulcanizedb
-COPY --from=builder /go/src/github.com/ipfs/go-ipfs/ipfs ipfs
-# XXX dir is already writeable RUN touch vulcanizedb.log
-
-CMD ["./startup_script.sh"]

dockerfiles/resync/docker-compose.yml

@@ -1,59 +0,0 @@
-version: '3.2'
-
-services:
-  db:
-    restart: always
-    image: postgres:10.12-alpine
-    environment:
-      POSTGRES_USER: "vdbm"
-      POSTGRES_DB: "vulcanize_public"
-      POSTGRES_PASSWORD: "password"
-    volumes:
-     - vulcanizedb_db_data:/var/lib/postgresql/data
-    expose:
-     - "5432"
-    ports:
-     - "127.0.0.1:8079:5432"
-
-  migrations:
-    restart: on-failure
-    depends_on:
-     - db
-    build:
-      context: ./../../
-      cache_from:
-        - alpine:latest
-      dockerfile: ./dockerfiles/migrations/Dockerfile
-    environment:
-      DATABASE_NAME: "vulcanize_public"
-      DATABASE_HOSTNAME: "db"
-      DATABASE_PORT: 5432
-      DATABASE_USER: "vdbm"
-      DATABASE_PASSWORD: "password"
-
-  resync:
-    depends_on:
-     - db
-     - migrations
-    build:
-      context: ./../../
-      cache_from:
-        - alpine:latest
-        - golang:1.12.4
-      dockerfile: ./dockerfiles/super_node/Dockerfile
-      args:
-        USER: "vdbm"
-        CONFIG_FILE: ./environments/superNodeBTC.toml
-    environment:
-      IPFS_INIT: "true"
-      DATABASE_NAME: "vulcanize_public"
-      DATABASE_HOSTNAME: "db"
-      DATABASE_PORT: 5432
-      DATABASE_USER: "vdbm"
-      DATABASE_PASSWORD: "password"
-    ports:
-     - "127.0.0.1:8082:8082"
-     - "127.0.0.1:8083:8083"
-
-volumes:
-  vulcanizedb_db_data:

dockerfiles/resync/startup_script.sh

@@ -1,50 +0,0 @@
-#!/bin/sh
-# Runs the db migrations and starts the super node services
-
-# Exit if the variable tests fail
-set -e
-set +x
-
-# Check the database variables are set
-test $DATABASE_HOSTNAME
-test $DATABASE_NAME
-test $DATABASE_PORT
-test $DATABASE_USER
-test $DATABASE_PASSWORD
-test $IPFS_INIT
-set +e
-
-# Export our database variables so that the IPFS Postgres plugin can use them
-export IPFS_PGHOST=$DATABASE_HOSTNAME
-export IPFS_PGUSER=$DATABASE_USER
-export IPFS_PGDATABASE=$DATABASE_NAME
-export IPFS_PGPORT=$DATABASE_PORT
-export IPFS_PGPASSWORD=$DATABASE_PASSWORD
-
-# If IPFS_INIT is true
-if [[ "$IPFS_INIT" = true ]] ; then
-    # initialize PG-IPFS
-    echo "Initializing Postgres-IPFS profile"
-    ./ipfs init --profile=postgresds
-else
-    echo "IPFS profile already initialized, skipping initialization"
-fi
-
-# If IPFS initialization was successful
-if [[ $? -eq 0 ]]; then
-    echo "Beginning the vulcanizedb super node process"
-    ./vulcanizedb resync --config=config.toml 2>&1 | tee -a vulcanizedb.log &
-else
-    echo "Could not initialize IPFS."
-    exit 1
-fi
-
-# If Vulcanizedb startup was successful
-if [ $? -eq 0 ]; then
-    echo "Resync successfully booted"
-else
-    echo "Could not start vulcanizedb resync process. Is the config file correct?"
-    exit 1
-fi
-
-tail -f vulcanizedb.log

dockerfiles/rinkeby/Dockerfile

@@ -1,9 +0,0 @@
-FROM golang:1.10.3-alpine3.7
-
-RUN apk add --no-cache make gcc musl-dev
-
-ADD . /go/src/github.com/vulcanize/vulcanizedb
-WORKDIR /go/src/github.com/vulcanize/vulcanizedb
-RUN go build -o /app main.go
-
-ENTRYPOINT ["/app"]

dockerfiles/rinkeby/config.toml

@@ -1,9 +0,0 @@
-[database]
-name = "vulcanizedb"
-hostname = "postgres"
-port = 5432
-user = "postgres"
-password = "postgres"
-
-[client]
-ipcPath = "/geth/geth.ipc"

dockerfiles/rinkeby/docker-compose.yml

@@ -1,63 +0,0 @@
-version: '2.2'
-
-services:
-
-  vulcanizedb:
-    build:
-      context: ./../../
-      dockerfile: dockerfiles/rinkeby/Dockerfile
-    container_name: rinkeby_vulcanizedb
-    command: "sync --starting-block-number 0 --config /config.toml"
-    volumes:
-      - "./config.toml:/config.toml"
-      - "vulcanizedb_geth_data:/geth"
-    networks:
-      vulcanizedb_net:
-
-  migrations:
-    image: migrate/migrate:v3.3.0
-    container_name: rinkeby_vulcanizedb_migrations
-    depends_on:
-      postgres:
-        condition: service_healthy
-    command: -database postgresql://postgres:postgres@postgres:5432/vulcanizedb?sslmode=disable -path /migrations up
-    volumes:
-      - ./../../db/migrations:/migrations
-    networks:
-      vulcanizedb_net:
-
-  postgres:
-    image: postgres:9.6.5-alpine
-    container_name: rinkeby_vulcanizedb_postgres
-    environment:
-      POSTGRES_USER: postgres
-      POSTGRES_DB: vulcanizedb
-      POSTGRES_PASSWORD: postgres
-    volumes:
-      - "vulcanizedb_db_data:/var/lib/postgresql/data"
-    networks:
-      vulcanizedb_net:
-    healthcheck:
-      test: ["CMD", "pg_isready"]
-      interval: 5s
-      timeout: 5s
-      retries: 30
-  
-  geth:
-    image: ethereum/client-go:v1.8.11
-    container_name: rinkeby_vulcanizedb_geth
-    cpus: 0.3
-    hostname: eth
-    command: '--rinkeby --rpc --rpcaddr="0.0.0.0" --rpcvhosts="geth"'
-    volumes:
-      - "vulcanizedb_geth_data:/root/.ethereum/rinkeby"
-    networks:
-      vulcanizedb_net:
-
-volumes:
-  vulcanizedb_geth_data:
-  vulcanizedb_db_data:
-
-networks:
-  vulcanizedb_net:
-    driver: bridge

dockerfiles/super_node/docker-compose.yml

@@ -15,28 +15,9 @@ services:
     ports:
      - "127.0.0.1:8079:5432"
 
-  migrations:
-    restart: on-failure
-    depends_on:
-      - db
-    build:
-      context: ./../../
-      cache_from:
-       - alpine:latest
-      dockerfile: ./dockerfiles/migrations/Dockerfile
-      args:
-        USER: "vdbm"
-    environment:
-      DATABASE_NAME: "vulcanize_public"
-      DATABASE_HOSTNAME: "db"
-      DATABASE_PORT: 5432
-      DATABASE_USER: "vdbm"
-      DATABASE_PASSWORD: "password"
-
   btc:
     depends_on:
       - db
-      - migrations
     build:
       context: ./../../
       cache_from:
@@ -47,7 +28,9 @@ services:
         USER: "vdbm"
         CONFIG_FILE: ./environments/superNodeBTC.toml
     environment:
+      VDB_COMMAND: "superNode"
       IPFS_INIT: "true"
+      IPFS_PATH: "/root/.btc/.ipfs"
       DATABASE_NAME: "vulcanize_public"
       DATABASE_HOSTNAME: "db"
       DATABASE_PORT: 5432
@@ -60,7 +43,6 @@ services:
   eth:
     depends_on:
       - db
-      - migrations
     build:
       context: ./../../
       cache_from:
@@ -71,7 +53,9 @@ services:
         USER: "vdbm"
         CONFIG_FILE: ./environments/superNodeETH.toml
     environment:
+      VDB_COMMAND: "superNode"
       IPFS_INIT: "true"
+      IPFS_PATH: "/root/.eth/.ipfs"
       DATABASE_NAME: "vulcanize_public"
       DATABASE_HOSTNAME: "db"
       DATABASE_PORT: 5432
@@ -81,11 +65,32 @@ services:
      - "127.0.0.1:8080:8080"
      - "127.0.0.1:8081:8081"
 
+  resync:
+    depends_on:
+    - db
+    build:
+      context: ./../../
+      cache_from:
+      - alpine:latest
+      - golang:1.12.4
+      dockerfile: ./dockerfiles/super_node/Dockerfile
+      args:
+        USER: "vdbm"
+        CONFIG_FILE: ./environments/superNodeETH.toml
+    environment:
+      VDB_COMMAND: "resync"
+      IPFS_INIT: "true"
+      IPFS_PATH: "/root/.eth_resync/.ipfs"
+      DATABASE_NAME: "vulcanize_public"
+      DATABASE_HOSTNAME: "db"
+      DATABASE_PORT: 5432
+      DATABASE_USER: "vdbm"
+      DATABASE_PASSWORD: "password"
+
   graphql:
     restart: always
     depends_on:
       - db
-      - migrations
     build:
       context: ./../../
       cache_from:

dockerfiles/super_node/startup_script.sh

@@ -12,6 +12,8 @@ test $DATABASE_PORT
 test $DATABASE_USER
 test $DATABASE_PASSWORD
 test $IPFS_INIT
+test $IPFS_PATH
+test $VDB_COMMAND
 set +e
 
 # Export our database variables so that the IPFS Postgres plugin can use them
@@ -21,19 +23,34 @@ export IPFS_PGDATABASE=$DATABASE_NAME
 export IPFS_PGPORT=$DATABASE_PORT
 export IPFS_PGPASSWORD=$DATABASE_PASSWORD
 
-# If IPFS_INIT is true
+# Construct the connection string for postgres
-if [[ "$IPFS_INIT" = true ]] ; then
+VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable
-    # initialize PG-IPFS
+
-    echo "Initializing Postgres-IPFS profile"
+# Run the DB migrations
-    ./ipfs init --profile=postgresds
+echo "Connecting with: $VDB_PG_CONNECT"
+echo "Running database migrations"
+./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up
+
+
+# If the db migrations ran without err
+if [[ $? -eq 0 ]]; then
+    # and IPFS_INIT is true
+    if [[ "$IPFS_INIT" = true ]] ; then
+        # initialize PG-IPFS
+        echo "Initializing Postgres-IPFS profile"
+        ./ipfs init --profile=postgresds
+    else
+        echo "IPFS profile already initialized, skipping initialization"
+    fi
 else
-    echo "IPFS profile already initialized, skipping initialization"
+    echo "Could not run migrations. Are the database details correct?"
+    exit 1
 fi
 
 # If IPFS initialization was successful
 if [[ $? -eq 0 ]]; then
     echo "Beginning the vulcanizedb super node process"
-    ./vulcanizedb superNode --config=config.toml 2>&1 | tee -a vulcanizedb.log &
+    ./vulcanizedb ${VDB_COMMAND} --config=config.toml 2>&1 | tee -a vulcanizedb.log &
 else
     echo "Could not initialize IPFS."
     exit 1
@@ -47,4 +64,4 @@ else
     exit 1
 fi
 
-tail -f vulcanizedb.log
+tail -f vulcanizedb.log

environments/superNodeBTC.toml

@@ -6,7 +6,7 @@
     password = "" # $DATABASE_PASSWORD
 
 [ipfs]
-    path = "/root/.ipfs" # $IPFS_PATH
+    path = "/root/.btc/.ipfs" # $IPFS_PATH
 
 [resync]
     chain = "bitcoin" # $RESYNC_CHAIN

environments/superNodeETH.toml

@@ -6,7 +6,7 @@
     password = "" # $DATABASE_PASSWORD
 
 [ipfs]
-    path = "/root/.ipfs" # $IPFS_PATH
+    path = "/root/.eth/.ipfs" # $IPFS_PATH
 
 [resync]
     chain = "ethereum" # $RESYNC_CHAIN