Merge pull request #6 from vulcanize/docker

Docker compose and CI

.github/workflows/on-master.yaml

@@ -0,0 +1,25 @@
+name: Docker Compose Build
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  build:
+    name: Run docker build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Get the version
+        id: vars
+        run: echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
+      - name: Run docker build
+        run: make docker-build
+      - name: Tag docker image
+        run: docker tag vulcanize/ipld-eth-server docker.pkg.github.com/vulcanize/ipld-eth-server/ipld-eth-server:${{steps.vars.outputs.sha}}
+      - name: Docker Login
+        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
+      - name: Docker Push
+        run: docker push docker.pkg.github.com/vulcanize/ipld-eth-server/ipld-eth-server:${{steps.vars.outputs.sha}}
+

.github/workflows/on-pr.yaml

@@ -0,0 +1,12 @@
+name: Docker Build
+
+on: [pull_request]
+
+jobs:
+  build:
+    name: Run docker build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Run docker build
+        run: make docker-build

.github/workflows/publish.yaml

@@ -0,0 +1,25 @@
+name: Publish Docker image
+on:
+  release:
+    types: [published]
+jobs:
+  push_to_registries:
+    name: Push Docker image to Docker Hub
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get the version
+        id: vars
+        run: |
+          echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
+          echo ::set-output name=tag::$(echo ${GITHUB_REF#refs/tags/})
+      - name: Docker Login to Github Registry
+        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
+      - name: Docker Pull
+        run: docker pull docker.pkg.github.com/vulcanize/ipld-eth-server/ipld-eth-server:${{steps.vars.outputs.sha}}
+      - name: Docker Login to Docker Registry
+        run: echo ${{ secrets.VULCANIZEJENKINS_PAT }} | docker login -u vulcanizejenkins --password-stdin
+      - name: Tag docker image
+        run: docker tag docker.pkg.github.com/vulcanize/ipld-eth-server/ipld-eth-server:${{steps.vars.outputs.sha}} vulcanize/ipld-eth-server:${{steps.vars.outputs.tag}}
+      - name: Docker Push to Docker Hub
+        run: docker push vulcanize/ipld-eth-server:${{steps.vars.outputs.tag}}
+

Dockerfile

@@ -4,26 +4,22 @@ RUN apk --update --no-cache add make git g++ linux-headers
 # DEBUG
 RUN apk add busybox-extras
 
-# Get and build ipld-eth-server
+# Build ipld-eth-server
-ADD . /go/src/github.com/vulcanize/ipld-eth-server
 WORKDIR /go/src/github.com/vulcanize/ipld-eth-server
+ADD . .
 RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o ipld-eth-server .
 
-# Build migration tool
+# Copy migration tool
 WORKDIR /
-RUN go get -u -d github.com/pressly/goose/cmd/goose
+ARG GOOSE_VER="v2.6.0"
-WORKDIR /go/src/github.com/pressly/goose/cmd/goose
+ADD https://github.com/pressly/goose/releases/download/${GOOSE_VER}/goose-linux64 ./goose
-RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -tags='no_mysql no_sqlite' -o goose .
+RUN chmod +x ./goose
-
-WORKDIR /go/src/github.com/vulcanize/ipld-eth-server
 
 # app container
 FROM alpine
 
-ARG USER
+ARG USER="vdm"
-ARG CONFIG_FILE
+ARG CONFIG_FILE="./environments/example.toml"
-ARG EXPOSE_PORT_1
-ARG EXPOSE_PORT_2
 
 RUN adduser -Du 5000 $USER
 WORKDIR /app
@@ -33,17 +29,13 @@ USER $USER
 # chown first so dir is writable
 # note: using $USER is merged, but not in the stable release yet
 COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/ipld-eth-server/$CONFIG_FILE config.toml
-COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/ipld-eth-server/dockerfiles/super_node/startup_script.sh .
+COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/ipld-eth-server/entrypoint.sh .
-COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/ipld-eth-server/dockerfiles/super_node/entrypoint.sh .
 
 
 # keep binaries immutable
 COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-server/ipld-eth-server ipld-eth-server
-COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose goose
+COPY --from=builder /goose goose
 COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-server/db/migrations migrations/vulcanizedb
 COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-server/environments environments
 
-EXPOSE $EXPOSE_PORT_1
+ENTRYPOINT ["/app/entrypoint.sh"]
-EXPOSE $EXPOSE_PORT_2
-
-ENTRYPOINT ["/app/startup_script.sh"]

Makefile

@@ -137,3 +137,8 @@ version_migrations:
 import:
 	test -n "$(NAME)" # $$NAME
 	psql $(NAME) < db/schema.sql
+
+## Build docker image
+.PHONY: docker-build
+docker-build:
+	docker build -t vulcanize/ipld-eth-server .

docker-compose.yml

@@ -0,0 +1,78 @@
+version: '3.2'
+
+services:
+  dapptools:
+    restart: unless-stopped
+    image: vulcanize/dapptools:v0.29.0-statediff-0.0.2
+    ports:
+      - "127.0.0.1:8545:8545"
+      - "127.0.0.1:8546:8546"
+
+  db:
+    restart: always
+    image: postgres:10.12-alpine
+    environment:
+      POSTGRES_USER: "vdbm"
+      POSTGRES_DB: "vulcanize_public"
+      POSTGRES_PASSWORD: "password"
+    volumes:
+     - vdb_db_eth_server:/var/lib/postgresql/data
+    ports:
+     - "127.0.0.1:8077:5432"
+
+  eth-indexer:
+    restart: unless-stopped
+    depends_on:
+      - db
+      - dapptools
+    image: vulcanize/ipld-eth-indexer:v0.3.0-alpha
+    environment:
+      DATABASE_NAME: vulcanize_public
+      DATABASE_HOSTNAME: db
+      DATABASE_PORT: 5432
+      DATABASE_USER: vdbm
+      DATABASE_PASSWORD: password
+      ETH_WS_PATH: "dapptools:8546"
+      ETH_HTTP_PATH: "dapptools:8545"
+      ETH_CHAIN_ID: 4
+      ETH_NETWORK_ID: 4
+      VDB_COMMAND: sync
+
+  eth-server:
+    depends_on:
+      - db
+    build:
+      context: ./
+      cache_from:
+       - alpine:latest
+       - golang:1.13-alpine
+    environment:
+      VDB_COMMAND: "serve"
+      DATABASE_NAME: "vulcanize_public"
+      DATABASE_HOSTNAME: "db"
+      DATABASE_PORT: 5432
+      DATABASE_USER: "vdbm"
+      DATABASE_PASSWORD: "password"
+      SERVER_WS_PATH: "0.0.0.0:8081"
+      SERVER_HTTP_PATH: "0.0.0.0:8082"
+    ports:
+     - "127.0.0.1:8080:8080"
+     - "127.0.0.1:8081:8081"
+
+  graphql:
+    restart: unless-stopped
+    depends_on:
+      - db
+    image: vulcanize/postgraphile:v1.0.1
+    environment:
+      - PG_HOST=db
+      - PG_PORT=5432
+      - PG_DATABASE=vulcanize_public
+      - PG_USER=vdbm
+      - PG_PASSWORD=password
+      - SCHEMA=public,eth
+    ports:
+      - "127.0.0.1:5000:5000"
+
+volumes:
+  vdb_db_eth_server:

dockerfiles/migrations/Dockerfile

@@ -1,40 +0,0 @@
-FROM golang:alpine
-
-RUN apk --update --no-cache add make git g++ linux-headers
-# DEBUG
-RUN apk add busybox-extras
-
-# this is probably a noob move, but I want apk from alpine for the above but need to avoid Go 1.13 below as this error still occurs https://github.com/ipfs/go-ipfs/issues/6603
-FROM golang:1.12.4 as builder
-
-# Get and build ipld-eth-server
-ADD . /go/src/github.com/vulcanize/ipld-eth-server
-
-# Build migration tool
-RUN go get -u -d github.com/pressly/goose/cmd/goose
-WORKDIR /go/src/github.com/pressly/goose/cmd/goose
-RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -tags='no_mysql no_sqlite' -o goose .
-
-WORKDIR /go/src/github.com/vulcanize/ipld-eth-server
-
-# app container
-FROM alpine
-
-ARG USER
-
-RUN adduser -Du 5000 $USER
-WORKDIR /app
-RUN chown $USER /app
-USER $USER
-
-# chown first so dir is writable
-# note: using $USER is merged, but not in the stable release yet
-COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/ipld-eth-server/dockerfiles/migrations/startup_script.sh .
-
-
-# keep binaries immutable
-COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose goose
-COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-server/db/migrations migrations/vulcanizedb
-# XXX dir is already writeable RUN touch vulcanizedb.log
-
-CMD ["./startup_script.sh"]

dockerfiles/postgraphile/Dockerfile

@@ -1,8 +0,0 @@
-FROM node:alpine
-
-RUN npm install -g postgraphile
-RUN npm install -g postgraphile-plugin-connection-filter
-RUN npm install -g @graphile/pg-pubsub
-
-EXPOSE 5000
-ENTRYPOINT ["postgraphile"]

dockerfiles/postgraphile/docker-compose.yml

@@ -1,60 +0,0 @@
-version: '3.2'
-
-services:
-  db:
-    restart: always
-    image: postgres:10.12-alpine
-    environment:
-      POSTGRES_USER: "vdbm"
-      POSTGRES_DB: "vulcanize_public"
-      POSTGRES_PASSWORD: "password"
-    volumes:
-     - vulcanizedb_db_data:/var/lib/postgresql/data
-    expose:
-     - "5432"
-    ports:
-     - "127.0.0.1:8079:5432"
-
-  migrations:
-    restart: on-failure
-    depends_on:
-     - db
-    build:
-      context: ./../../
-      cache_from:
-        - alpine:latest
-      dockerfile: ./dockerfiles/migrations/Dockerfile
-      args:
-        USER: "vdbm"
-    environment:
-      DATABASE_NAME: "vulcanize_public"
-      DATABASE_HOSTNAME: "db"
-      DATABASE_PORT: 5432
-      DATABASE_USER: "vdbm"
-      DATABASE_PASSWORD: "password"
-
-  graphql:
-    restart: always
-    depends_on:
-     - db
-     - migrations
-    build:
-      context: ./../../
-      cache_from:
-       - node:alpine
-      dockerfile: ./dockerfiles/postgraphile/Dockerfile
-    expose:
-     - "5000"
-    ports:
-     - "127.0.0.1:5000:5000"
-    command: ["--plugins", "@graphile/pg-pubsub",
-              "--subscriptions",
-              "--simple-subscriptions",
-              "--connection", "postgres://vdbm:password@db:5432/vulcanize_public",
-              "--port", "5000",
-              "-n", "0.0.0.0",
-              "--schema", "public,btc,eth",
-              "--append-plugins", "postgraphile-plugin-connection-filter"]
-
-volumes:
-  vulcanizedb_db_data:

dockerfiles/super_node/docker-compose.yml

@@ -1,87 +0,0 @@
-version: '3.2'
-
-services:
-  db:
-    restart: always
-    image: postgres:10.12-alpine
-    environment:
-      POSTGRES_USER: "vdbm"
-      POSTGRES_DB: "vulcanize_public"
-      POSTGRES_PASSWORD: "password"
-    volumes:
-     - vulcanizedb_db_data:/var/lib/postgresql/data
-    expose:
-     - "5432"
-    ports:
-     - "127.0.0.1:8079:5432"
-
-  btc:
-    depends_on:
-      - db
-    build:
-      context: ./../../
-      cache_from:
-       - alpine:latest
-       - golang:1.12.4
-      dockerfile: ./dockerfiles/super_node/Dockerfile
-      args:
-        USER: "vdbm"
-        CONFIG_FILE: ./environments/superNodeBTC.toml
-    environment:
-      VDB_COMMAND: "watcher"
-      DATABASE_NAME: "vulcanize_public"
-      DATABASE_HOSTNAME: "db"
-      DATABASE_PORT: 5432
-      DATABASE_USER: "vdbm"
-      DATABASE_PASSWORD: "password"
-    ports:
-     - "127.0.0.1:8082:8082"
-     - "127.0.0.1:8083:8083"
-
-  eth:
-    depends_on:
-      - db
-    build:
-      context: ./../../
-      cache_from:
-       - alpine:latest
-       - golang:1.12.4
-      dockerfile: ./dockerfiles/super_node/Dockerfile
-      args:
-        USER: "vdbm"
-        CONFIG_FILE: ./environments/superNodeETH.toml
-    environment:
-      VDB_COMMAND: "watch"
-      DATABASE_NAME: "vulcanize_public"
-      DATABASE_HOSTNAME: "db"
-      DATABASE_PORT: 5432
-      DATABASE_USER: "vdbm"
-      DATABASE_PASSWORD: "password"
-    ports:
-     - "127.0.0.1:8080:8080"
-     - "127.0.0.1:8081:8081"
-
-  graphql:
-    restart: always
-    depends_on:
-      - db
-    build:
-      context: ./../../
-      cache_from:
-      - node:alpine
-      dockerfile: ./dockerfiles/postgraphile/Dockerfile
-    expose:
-      - "5000"
-    ports:
-      - "127.0.0.1:5000:5000"
-    command: ["--plugins", "@graphile/pg-pubsub",
-              "--subscriptions",
-              "--simple-subscriptions",
-              "--connection", "postgres://vdbm:password@db:5432/vulcanize_public",
-              "--port", "5000",
-              "-n", "0.0.0.0",
-              "--schema", "public,btc,eth",
-              "--append-plugins", "postgraphile-plugin-connection-filter"]
-
-volumes:
-  vulcanizedb_db_data:

dockerfiles/super_node/entrypoint.sh

@@ -1,59 +0,0 @@
-#!/bin/sh
-# Runs the db migrations and starts the watcher services
-
-# Exit if the variable tests fail
-set -e
-set +x
-
-# Check the database variables are set
-# XXX set defaults, don't silently fail
-#test $DATABASE_HOSTNAME
-#test $DATABASE_NAME
-#test $DATABASE_PORT
-#test $DATABASE_USER
-#test $DATABASE_PASSWORD
-#test $IPFS_INIT
-VDB_COMMAND=${VDB_COMMAND:-watch}
-set +e
-
-# Construct the connection string for postgres
-VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable
-
-# Run the DB migrations
-echo "Connecting with: $VDB_PG_CONNECT"
-echo "Running database migrations"
-./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up
-rv=$?
-
-if [ $rv != 0 ]; then
-  echo "Could not run migrations. Are the database details correct?"
-  exit 1
-fi
-
-
-echo "Beginning the vulcanizedb process"
-VDB_CONFIG_FILE=${VDB_CONFIG_FILE:-config.toml}
-DEFAULT_OPTIONS="--config=$VDB_CONFIG_FILE"
-VDB_FULL_CL=${VDB_FULL_CL:-$VDB_COMMAND $DEFAULT_OPTIONS}
-echo running: ./ipld-eth-server $VDB_FULL_CL $@
-
-case "$1" in
-  "/bin/sh" )
-    echo dropping to shell
-    exec /bin/sh
-esac
-
-vdb_args="$@"
-# default is to use the config passed by the build arg
-if [[ -z "$vdb_args" ]]; then
-  vdb_args="--config=config.toml"
-fi
-
-echo running: ./ipld-eth-server $vdb_args
-./ipld-eth-server $vdb_args
-rv=$?
-
-if [ $rv != 0 ]; then
-  echo "VulcanizeDB startup failed"
-  exit 1
-fi

dockerfiles/super_node/startup_script.sh

@@ -1,64 +0,0 @@
-#!/bin/sh
-# Runs the db migrations and starts the watcher services
-
-# Exit if the variable tests fail
-set -e
-set +x
-
-# Check the database variables are set
-test $DATABASE_HOSTNAME
-test $DATABASE_NAME
-test $DATABASE_PORT
-test $DATABASE_USER
-test $DATABASE_PASSWORD
-test $IPFS_INIT
-test $VDB_COMMAND
-set +e
-
-# Export our database variables so that the IPFS Postgres plugin can use them
-export IPFS_PGHOST=$DATABASE_HOSTNAME
-export IPFS_PGUSER=$DATABASE_USER
-export IPFS_PGDATABASE=$DATABASE_NAME
-export IPFS_PGPORT=$DATABASE_PORT
-export IPFS_PGPASSWORD=$DATABASE_PASSWORD
-
-# Construct the connection string for postgres
-VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable
-
-# Run the DB migrations
-echo "Connecting with: $VDB_PG_CONNECT"
-echo "Running database migrations"
-./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up
-
-
-# If the db migrations ran without err
-if [[ $? -eq 0 ]]; then
-    # and IPFS_INIT is true
-    if [[ "$IPFS_INIT" = true ]] ; then
-        # initialize PG-IPFS
-        echo "Initializing Postgres-IPFS profile"
-        ./ipfs init --profile=postgresds
-    else
-        echo "IPFS profile already initialized, skipping initialization"
-    fi
-else
-    echo "Could not run migrations. Are the database details correct?"
-    exit 1
-fi
-
-# If IPFS initialization was successful
-if [[ $? -eq 0 ]]; then
-    echo "Running the VulcanizeDB process"
-    ./ipld-eth-server ${VDB_COMMAND} --config=config.toml
-else
-    echo "Could not initialize IPFS."
-    exit 1
-fi
-
-# If VulcanizeDB process was successful
-if [ $? -eq 0 ]; then
-    echo "VulcanizeDB process ran successfully"
-else
-    echo "Could not start VulcanizeDB process. Is the config file correct?"
-    exit 1
-fi

entrypoint.sh

@@ -1,18 +1,6 @@
 #!/bin/sh
 # Runs the db migrations and starts the watcher services
 
-# Exit if the variable tests fail
-set -e
-set +x
-
-# Check the database variables are set
-test $DATABASE_HOSTNAME
-test $DATABASE_NAME
-test $DATABASE_PORT
-test $DATABASE_USER
-test $DATABASE_PASSWORD
-set +e
-
 # Construct the connection string for postgres
 VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable
 
@@ -20,13 +8,21 @@ VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME
 echo "Connecting with: $VDB_PG_CONNECT"
 echo "Running database migrations"
 ./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up
+rv=$?
+
+if [ $rv != 0 ]; then
+  echo "Could not run migrations. Are the database details correct?"
+  exit 1
+fi
 
 
-# If the db migrations ran without err
+echo "Beginning the ipld-eth-server process"
-if [[ $? -eq 0 ]]; then
+
-    echo "Migrations ran successfully"
+echo running: ./ipld-eth-server ${VDB_COMMAND} --config=config.toml
-    exit 0
+./ipld-eth-server ${VDB_COMMAND} --config=config.toml
-else
+rv=$?
-    echo "Could not run migrations. Are the database details correct?"
+
-    exit 1
+if [ $rv != 0 ]; then
+  echo "ipld-eth-server startup failed"
+  exit 1
 fi

pkg/serve/config.go

@@ -60,12 +60,12 @@ func NewConfig() (*Config, error) {
 
 	c.DBConfig.Init()
 
-	wsPath := viper.GetString("watcher.wsPath")
+	wsPath := viper.GetString("server.wsPath")
 	if wsPath == "" {
 		wsPath = "127.0.0.1:8080"
 	}
 	c.WSEndpoint = wsPath
-	ipcPath := viper.GetString("watcher.ipcPath")
+	ipcPath := viper.GetString("server.ipcPath")
 	if ipcPath == "" {
 		home, err := os.UserHomeDir()
 		if err != nil {
@@ -74,7 +74,7 @@ func NewConfig() (*Config, error) {
 		ipcPath = filepath.Join(home, ".vulcanize/vulcanize.ipc")
 	}
 	c.IPCEndpoint = ipcPath
-	httpPath := viper.GetString("watcher.httpPath")
+	httpPath := viper.GetString("server.httpPath")
 	if httpPath == "" {
 		httpPath = "127.0.0.1:8081"
 	}