From 2d5bd2defcadfc0aee89117054ad14bfc9efeacb Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Wed, 18 Mar 2020 08:48:25 -0500 Subject: [PATCH] dockerfile for resync process --- dockerfiles/resync/Dockerfile | 54 +++++++++++++++++++ dockerfiles/resync/startup_script.sh | 65 +++++++++++++++++++++++ dockerfiles/super_node/docker-compose.yml | 28 +++++----- dockerfiles/super_node/startup_script.sh | 4 +- 4 files changed, 134 insertions(+), 17 deletions(-) create mode 100644 dockerfiles/resync/Dockerfile create mode 100644 dockerfiles/resync/startup_script.sh diff --git a/dockerfiles/resync/Dockerfile b/dockerfiles/resync/Dockerfile new file mode 100644 index 00000000..8b8995f5 --- /dev/null +++ b/dockerfiles/resync/Dockerfile @@ -0,0 +1,54 @@ +FROM golang:alpine + +RUN apk --update --no-cache add make git g++ linux-headers +# DEBUG +RUN apk add busybox-extras + +# this is probably a noob move, but I want apk from alpine for the above but need to avoid Go 1.13 below as this error still occurs https://github.com/ipfs/go-ipfs/issues/6603 +FROM golang:1.12.4 as builder + +# Get and build vulcanizedb +ADD . /go/src/github.com/vulcanize/vulcanizedb +WORKDIR /go/src/github.com/vulcanize/vulcanizedb +RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o vulcanizedb . + +# Get and build vulcanize's go-ipfs fork +RUN go get -u -d github.com/ipfs/go-ipfs +WORKDIR /go/src/github.com/ipfs/go-ipfs +RUN git remote add vulcanize https://github.com/vulcanize/go-ipfs.git +RUN git fetch vulcanize +RUN git checkout -b pg_ipfs vulcanize/postgres_update +RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o ipfs ./cmd/ipfs + +# Build migration tool +RUN go get -u -d github.com/pressly/goose/cmd/goose +WORKDIR /go/src/github.com/pressly/goose/cmd/goose +RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -tags='no_mysql no_sqlite' -o goose . + +WORKDIR /go/src/github.com/vulcanize/vulcanizedb + +# app container +FROM alpine + +ARG USER +ARG CONFIG_FILE + +RUN adduser -Du 5000 $USER +WORKDIR /app +RUN chown $USER /app +USER $USER + +# chown first so dir is writable +# note: using $USER is merged, but not in the stable release yet +COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/vulcanizedb/$CONFIG_FILE config.toml +COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/vulcanizedb/dockerfiles/resync/startup_script.sh . + + +# keep binaries immutable +COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/vulcanizedb vulcanizedb +COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose goose +COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/db/migrations migrations/vulcanizedb +COPY --from=builder /go/src/github.com/ipfs/go-ipfs/ipfs ipfs +# XXX dir is already writeable RUN touch vulcanizedb.log + +CMD ["./startup_script.sh"] \ No newline at end of file diff --git a/dockerfiles/resync/startup_script.sh b/dockerfiles/resync/startup_script.sh new file mode 100644 index 00000000..ddeec23f --- /dev/null +++ b/dockerfiles/resync/startup_script.sh @@ -0,0 +1,65 @@ +#!/bin/sh +# Runs the db migrations and starts the super node services + +# Exit if the variable tests fail +set -e +set +x + +# Check the database variables are set +test $DATABASE_HOSTNAME +test $DATABASE_NAME +test $DATABASE_PORT +test $DATABASE_USER +test $DATABASE_PASSWORD +test $IPFS_INIT +set +e + +# Export our database variables so that the IPFS Postgres plugin can use them +export IPFS_PGHOST=$DATABASE_HOSTNAME +export IPFS_PGUSER=$DATABASE_USER +export IPFS_PGDATABASE=$DATABASE_NAME +export IPFS_PGPORT=$DATABASE_PORT +export IPFS_PGPASSWORD=$DATABASE_PASSWORD + +# Construct the connection string for postgres +VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable + +# Run the DB migrations +echo "Connecting with: $VDB_PG_CONNECT" +echo "Running database migrations" +./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up + + +# If the db migrations ran without err +if [[ $? -eq 0 ]]; then + # and IPFS_INIT is true + if [[ "$IPFS_INIT" = true ]] ; then + # initialize PG-IPFS + echo "Initializing Postgres-IPFS profile" + ./ipfs init --profile=postgresds + else + echo "IPFS profile already initialized, skipping initialization" + fi +else + echo "Could not run migrations. Are the database details correct?" + exit +fi + +# If IPFS initialization was successful +if [[ $? -eq 0 ]]; then + echo "Beginning the vulcanizedb super node process" + ./vulcanizedb resync --config=config.toml 2>&1 | tee -a vulcanizedb.log & +else + echo "Could not initialize IPFS." + exit +fi + +# If Vulcanizedb startup was successful +if [ $? -eq 0 ]; then + echo "Resync successfully booted" +else + echo "Could not start vulcanizedb resync process. Is the config file correct?" + exit +fi + +tail -f vulcanizedb.log \ No newline at end of file diff --git a/dockerfiles/super_node/docker-compose.yml b/dockerfiles/super_node/docker-compose.yml index c17f0869..f8995549 100644 --- a/dockerfiles/super_node/docker-compose.yml +++ b/dockerfiles/super_node/docker-compose.yml @@ -28,11 +28,11 @@ services: CONFIG_FILE: ./environments/superNodeBTC.toml environment: IPFS_INIT: "true" - VDB_PG_NAME: "vulcanize_public" - VDB_PG_HOSTNAME: "db" - VDB_PG_PORT: 5432 - VDB_PG_USER: "postgres" - VDB_PG_PASSWORD: "password" + DATABASE_NAME: "vulcanize_public" + DATABASE_HOSTNAME: "db" + DATABASE_PORT: 5432 + DATABASE_USER: "postgres" + DATABASE_PASSWORD: "password" ports: - "127.0.0.1:8082:8082" - "127.0.0.1:8083:8083" @@ -50,11 +50,11 @@ services: CONFIG_FILE: ./environments/superNodeETH.toml environment: IPFS_INIT: "true" - VDB_PG_NAME: "vulcanize_public" - VDB_PG_HOSTNAME: "db" - VDB_PG_PORT: 5432 - VDB_PG_USER: "postgres" - VDB_PG_PASSWORD: "password" + DATABASE_NAME: "vulcanize_public" + DATABASE_HOSTNAME: "db" + DATABASE_PORT: 5432 + DATABASE_USER: "postgres" + DATABASE_PASSWORD: "password" ports: - "127.0.0.1:8080:8080" - "127.0.0.1:8081:8081" @@ -68,8 +68,6 @@ services: cache_from: - node:alpine dockerfile: ./dockerfiles/postgraphile/Dockerfile - environment: - DATABASE_URL: postgres://postgres:password@db:5432/vulcanize_public expose: - "5000" ports: @@ -77,10 +75,10 @@ services: command: ["--plugins", "@graphile/pg-pubsub", "--subscriptions", "--simple-subscriptions", - "--connection", $DATABASE_URL, + "--connection", "postgres://postgres:password@db:5432/vulcanize_public", "--port", "5000", - "-n", "0.0.0.0" - "--schema", "public,btc,eth" + "-n", "0.0.0.0", + "--schema", "public,btc,eth", "--append-plugins", "postgraphile-plugin-connection-filter"] volumes: diff --git a/dockerfiles/super_node/startup_script.sh b/dockerfiles/super_node/startup_script.sh index a1d03dc7..06c04a2e 100755 --- a/dockerfiles/super_node/startup_script.sh +++ b/dockerfiles/super_node/startup_script.sh @@ -56,9 +56,9 @@ fi # If Vulcanizedb startup was successful if [ $? -eq 0 ]; then - echo "Seed node successfully booted" + echo "Super node successfully booted" else - echo "Could not start vulcanizedb syncPublishScreenAndServe process. Is the config file correct?" + echo "Could not start vulcanizedb super node process. Is the config file correct?" exit fi