cerc-io/ipld-eth-server
18
0
Fork 1
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases 37 Wiki Activity

split running migrations into separate compose step; docker-compose files for different processes

Browse Source
This commit is contained in:
Ian Norden 2020-03-18 12:21:29 -05:00
parent 6ded3a6c85
commit 1d4b37aca9
9 changed files with 286 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/resync.go
View File

@ -16,8 +16,6 @@
package cmd
import (
"fmt"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
@ -56,5 +54,5 @@ func rsyncCmdCommand() {
if err := rService.Resync(); err != nil {
logWithCommand.Fatal(err)
}
fmt.Printf("%s %s resync finished", rConfig.Chain.String(), rConfig.ResyncType.String())
logWithCommand.Infof("%s %s resync finished", rConfig.Chain.String(), rConfig.ResyncType.String())
}

40
dockerfiles/migrations/Dockerfile Normal file
View File

@ -0,0 +1,40 @@
FROM golang:alpine
RUN apk --update --no-cache add make git g++ linux-headers
# DEBUG
RUN apk add busybox-extras
# this is probably a noob move, but I want apk from alpine for the above but need to avoid Go 1.13 below as this error still occurs https://github.com/ipfs/go-ipfs/issues/6603
FROM golang:1.12.4 as builder
# Get and build vulcanizedb
ADD . /go/src/github.com/vulcanize/vulcanizedb
# Build migration tool
RUN go get -u -d github.com/pressly/goose/cmd/goose
WORKDIR /go/src/github.com/pressly/goose/cmd/goose
RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -tags='no_mysql no_sqlite' -o goose .
WORKDIR /go/src/github.com/vulcanize/vulcanizedb
# app container
FROM alpine
ARG USER
RUN adduser -Du 5000 $USER
WORKDIR /app
RUN chown $USER /app
USER $USER
# chown first so dir is writable
# note: using $USER is merged, but not in the stable release yet
COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/vulcanizedb/dockerfiles/migrations/startup_script.sh .
# keep binaries immutable
COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose goose
COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/db/migrations migrations/vulcanizedb
# XXX dir is already writeable RUN touch vulcanizedb.log
CMD ["./startup_script.sh"]

37
dockerfiles/migrations/docker-compose.yml Normal file
View File

@ -0,0 +1,37 @@
version: '3.2'
services:
db:
restart: always
image: postgres:10.12-alpine
environment:
POSTGRES_USER: "vdbm"
POSTGRES_DB: "vulcanize_public"
POSTGRES_PASSWORD: "password"
volumes:
- vulcanizedb_db_data:/var/lib/postgresql/data
expose:
- "5432"
ports:
- "127.0.0.1:8079:5432"
migrations:
restart: on-failure
depends_on:
- db
build:
context: ./../../
cache_from:
- alpine:latest
dockerfile: ./dockerfiles/migrations/Dockerfile
args:
USER: "vdbm"
environment:
DATABASE_NAME: "vulcanize_public"
DATABASE_HOSTNAME: "db"
DATABASE_PORT: 5432
DATABASE_USER: "vdbm"
DATABASE_PASSWORD: "password"
volumes:
vulcanizedb_db_data:

32
dockerfiles/migrations/startup_script.sh Executable file
View File

@ -0,0 +1,32 @@
#!/bin/sh
# Runs the db migrations and starts the super node services
# Exit if the variable tests fail
set -e
set +x
# Check the database variables are set
test $DATABASE_HOSTNAME
test $DATABASE_NAME
test $DATABASE_PORT
test $DATABASE_USER
test $DATABASE_PASSWORD
set +e
# Construct the connection string for postgres
VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable
# Run the DB migrations
echo "Connecting with: $VDB_PG_CONNECT"
echo "Running database migrations"
./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up
# If the db migrations ran without err
if [[ $? -eq 0 ]]; then
echo "Migrations ran successfully"
exit 0
else
echo "Could not run migrations. Are the database details correct?"
exit 1
fi

60
dockerfiles/postgraphile/docker-compose.yml Normal file
View File

@ -0,0 +1,60 @@
version: '3.2'
services:
db:
restart: always
image: postgres:10.12-alpine
environment:
POSTGRES_USER: "vdbm"
POSTGRES_DB: "vulcanize_public"
POSTGRES_PASSWORD: "password"
volumes:
- vulcanizedb_db_data:/var/lib/postgresql/data
expose:
- "5432"
ports:
- "127.0.0.1:8079:5432"
migrations:
restart: on-failure
depends_on:
- db
build:
context: ./../../
cache_from:
- alpine:latest
dockerfile: ./dockerfiles/migrations/Dockerfile
args:
USER: "vdbm"
environment:
DATABASE_NAME: "vulcanize_public"
DATABASE_HOSTNAME: "db"
DATABASE_PORT: 5432
DATABASE_USER: "vdbm"
DATABASE_PASSWORD: "password"
graphql:
restart: always
depends_on:
- db
- migrations
build:
context: ./../../
cache_from:
- node:alpine
dockerfile: ./dockerfiles/postgraphile/Dockerfile
expose:
- "5000"
ports:
- "127.0.0.1:5000:5000"
command: ["--plugins", "@graphile/pg-pubsub",
"--subscriptions",
"--simple-subscriptions",
"--connection", "postgres://vdbm:password@db:5432/vulcanize_public",
"--port", "5000",
"-n", "0.0.0.0",
"--schema", "public,btc,eth",
"--append-plugins", "postgraphile-plugin-connection-filter"]
volumes:
vulcanizedb_db_data:

59
dockerfiles/resync/docker-compose.yml Normal file
View File

@ -0,0 +1,59 @@
version: '3.2'
services:
db:
restart: always
image: postgres:10.12-alpine
environment:
POSTGRES_USER: "vdbm"
POSTGRES_DB: "vulcanize_public"
POSTGRES_PASSWORD: "password"
volumes:
- vulcanizedb_db_data:/var/lib/postgresql/data
expose:
- "5432"
ports:
- "127.0.0.1:8079:5432"
migrations:
restart: on-failure
depends_on:
- db
build:
context: ./../../
cache_from:
- alpine:latest
dockerfile: ./dockerfiles/migrations/Dockerfile
environment:
DATABASE_NAME: "vulcanize_public"
DATABASE_HOSTNAME: "db"
DATABASE_PORT: 5432
DATABASE_USER: "vdbm"
DATABASE_PASSWORD: "password"
resync:
depends_on:
- db
- migrations
build:
context: ./../../
cache_from:
- alpine:latest
- golang:1.12.4
dockerfile: ./dockerfiles/super_node/Dockerfile
args:
USER: "vdbm"
CONFIG_FILE: ./environments/superNodeBTC.toml
environment:
IPFS_INIT: "true"
DATABASE_NAME: "vulcanize_public"
DATABASE_HOSTNAME: "db"
DATABASE_PORT: 5432
DATABASE_USER: "vdbm"
DATABASE_PASSWORD: "password"
ports:
- "127.0.0.1:8082:8082"
- "127.0.0.1:8083:8083"
volumes:
vulcanizedb_db_data:

31
dockerfiles/resync/startup_script.sh Normal file → Executable file
View File

@ -21,28 +21,13 @@ export IPFS_PGDATABASE=$DATABASE_NAME
export IPFS_PGPORT=$DATABASE_PORT
export IPFS_PGPASSWORD=$DATABASE_PASSWORD
# Construct the connection string for postgres
VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable
# Run the DB migrations
echo "Connecting with: $VDB_PG_CONNECT"
echo "Running database migrations"
./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up
# If the db migrations ran without err
if [[ $? -eq 0 ]]; then
# and IPFS_INIT is true
if [[ "$IPFS_INIT" = true ]] ; then
# initialize PG-IPFS
echo "Initializing Postgres-IPFS profile"
./ipfs init --profile=postgresds
else
echo "IPFS profile already initialized, skipping initialization"
fi
# If IPFS_INIT is true
if [[ "$IPFS_INIT" = true ]] ; then
# initialize PG-IPFS
echo "Initializing Postgres-IPFS profile"
./ipfs init --profile=postgresds
else
echo "Could not run migrations. Are the database details correct?"
exit
echo "IPFS profile already initialized, skipping initialization"
fi
# If IPFS initialization was successful
@ -51,7 +36,7 @@ if [[ $? -eq 0 ]]; then
./vulcanizedb resync --config=config.toml 2>&1 | tee -a vulcanizedb.log &
else
echo "Could not initialize IPFS."
exit
exit 1
fi
# If Vulcanizedb startup was successful
@ -59,7 +44,7 @@ if [ $? -eq 0 ]; then
echo "Resync successfully booted"
else
echo "Could not start vulcanizedb resync process. Is the config file correct?"
exit
exit 1
fi
tail -f vulcanizedb.log

59
dockerfiles/super_node/docker-compose.yml
View File

@ -5,64 +5,87 @@ services:
restart: always
image: postgres:10.12-alpine
environment:
POSTGRES_USER: "postgres"
POSTGRES_USER: "vdbm"
POSTGRES_DB: "vulcanize_public"
POSTGRES_PASSWORD: "password"
volumes:
- vulcanizedb_db_data:/var/lib/postgresql/data
- vulcanizedb_db_data:/var/lib/postgresql/data
expose:
- "5432"
- "5432"
ports:
- "127.0.0.1:8079:5432"
- "127.0.0.1:8079:5432"
btc:
migrations:
restart: on-failure
depends_on:
- db
- db
build:
context: ./../../
cache_from:
- alpine:latest
- golang:1.12.4
- alpine:latest
dockerfile: ./dockerfiles/migrations/Dockerfile
args:
USER: "vdbm"
environment:
DATABASE_NAME: "vulcanize_public"
DATABASE_HOSTNAME: "db"
DATABASE_PORT: 5432
DATABASE_USER: "vdbm"
DATABASE_PASSWORD: "password"
btc:
depends_on:
- db
- migrations
build:
context: ./../../
cache_from:
- alpine:latest
- golang:1.12.4
dockerfile: ./dockerfiles/super_node/Dockerfile
args:
USER: "vdbm"
CONFIG_FILE: ./environments/superNodeBTC.toml
environment:
IPFS_INIT: "true"
DATABASE_NAME: "vulcanize_public"
DATABASE_HOSTNAME: "db"
DATABASE_PORT: 5432
DATABASE_USER: "postgres"
DATABASE_USER: "vdbm"
DATABASE_PASSWORD: "password"
ports:
- "127.0.0.1:8082:8082"
- "127.0.0.1:8083:8083"
- "127.0.0.1:8082:8082"
- "127.0.0.1:8083:8083"
eth:
depends_on:
- db
- db
- migrations
build:
context: ./../../
cache_from:
- alpine:latest
- golang:1.12.4
- alpine:latest
- golang:1.12.4
dockerfile: ./dockerfiles/super_node/Dockerfile
args:
USER: "vdbm"
CONFIG_FILE: ./environments/superNodeETH.toml
environment:
IPFS_INIT: "true"
DATABASE_NAME: "vulcanize_public"
DATABASE_HOSTNAME: "db"
DATABASE_PORT: 5432
DATABASE_USER: "postgres"
DATABASE_USER: "vdbm"
DATABASE_PASSWORD: "password"
ports:
- "127.0.0.1:8080:8080"
- "127.0.0.1:8081:8081"
- "127.0.0.1:8080:8080"
- "127.0.0.1:8081:8081"
graphql:
restart: always
depends_on:
- db
- migrations
build:
context: ./../../
cache_from:
@ -75,7 +98,7 @@ services:
command: ["--plugins", "@graphile/pg-pubsub",
"--subscriptions",
"--simple-subscriptions",
"--connection", "postgres://postgres:password@db:5432/vulcanize_public",
"--connection", "postgres://vdbm:password@db:5432/vulcanize_public",
"--port", "5000",
"-n", "0.0.0.0",
"--schema", "public,btc,eth",

31
dockerfiles/super_node/startup_script.sh
View File

@ -21,28 +21,13 @@ export IPFS_PGDATABASE=$DATABASE_NAME
export IPFS_PGPORT=$DATABASE_PORT
export IPFS_PGPASSWORD=$DATABASE_PASSWORD
# Construct the connection string for postgres
VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable
# Run the DB migrations
echo "Connecting with: $VDB_PG_CONNECT"
echo "Running database migrations"
./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up
# If the db migrations ran without err
if [[ $? -eq 0 ]]; then
# and IPFS_INIT is true
if [[ "$IPFS_INIT" = true ]] ; then
# initialize PG-IPFS
echo "Initializing Postgres-IPFS profile"
./ipfs init --profile=postgresds
else
echo "IPFS profile already initialized, skipping initialization"
fi
# If IPFS_INIT is true
if [[ "$IPFS_INIT" = true ]] ; then
# initialize PG-IPFS
echo "Initializing Postgres-IPFS profile"
./ipfs init --profile=postgresds
else
echo "Could not run migrations. Are the database details correct?"
exit
echo "IPFS profile already initialized, skipping initialization"
fi
# If IPFS initialization was successful
@ -51,7 +36,7 @@ if [[ $? -eq 0 ]]; then
./vulcanizedb superNode --config=config.toml 2>&1 | tee -a vulcanizedb.log &
else
echo "Could not initialize IPFS."
exit
exit 1
fi
# If Vulcanizedb startup was successful
@ -59,7 +44,7 @@ if [ $? -eq 0 ]; then
echo "Super node successfully booted"
else
echo "Could not start vulcanizedb super node process. Is the config file correct?"
exit
exit 1
fi
tail -f vulcanizedb.log