cerc-io/ipld-eth-server
18
0
Fork 1
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases 37 Wiki Activity

Docker compose and CI #6

Merged
ramilexe merged 4 commits from docker into master 2020-09-09 20:12:36 +00:00
Conversation 0 Commits 4 Files Changed 14 +174 -359
14 changed files with 174 additions and 359 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
.github/workflows/on-master.yaml vendored Normal file
View File

@ -0,0 +1,25 @@
name: Docker Compose Build
on:
push:
branches:
- master
jobs:
build:
name: Run docker build
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Get the version
id: vars
run: echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
- name: Run docker build
run: make docker-build
- name: Tag docker image
run: docker tag vulcanize/ipld-eth-server docker.pkg.github.com/vulcanize/ipld-eth-server/ipld-eth-server:${{steps.vars.outputs.sha}}
- name: Docker Login
run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
- name: Docker Push
run: docker push docker.pkg.github.com/vulcanize/ipld-eth-server/ipld-eth-server:${{steps.vars.outputs.sha}}

12
.github/workflows/on-pr.yaml vendored Normal file
View File

@ -0,0 +1,12 @@
name: Docker Build
on: [pull_request]
jobs:
build:
name: Run docker build
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Run docker build
run: make docker-build

25
.github/workflows/publish.yaml vendored Normal file
View File

@ -0,0 +1,25 @@
name: Publish Docker image
on:
release:
types: [published]
jobs:
push_to_registries:
name: Push Docker image to Docker Hub
runs-on: ubuntu-latest
steps:
- name: Get the version
id: vars
run: |
echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
echo ::set-output name=tag::$(echo ${GITHUB_REF#refs/tags/})
- name: Docker Login to Github Registry
run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
- name: Docker Pull
run: docker pull docker.pkg.github.com/vulcanize/ipld-eth-server/ipld-eth-server:${{steps.vars.outputs.sha}}
- name: Docker Login to Docker Registry
run: echo ${{ secrets.VULCANIZEJENKINS_PAT }} | docker login -u vulcanizejenkins --password-stdin
- name: Tag docker image
run: docker tag docker.pkg.github.com/vulcanize/ipld-eth-server/ipld-eth-server:${{steps.vars.outputs.sha}} vulcanize/ipld-eth-server:${{steps.vars.outputs.tag}}
- name: Docker Push to Docker Hub
run: docker push vulcanize/ipld-eth-server:${{steps.vars.outputs.tag}}

30
dockerfiles/super_node/Dockerfile → Dockerfile
View File

@ -4,26 +4,22 @@ RUN apk --update --no-cache add make git g++ linux-headers
# DEBUG
RUN apk add busybox-extras
# Get and build ipld-eth-server
ADD . /go/src/github.com/vulcanize/ipld-eth-server
# Build ipld-eth-server
WORKDIR /go/src/github.com/vulcanize/ipld-eth-server
ADD . .
RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o ipld-eth-server .
# Build migration tool
# Copy migration tool
WORKDIR /
RUN go get -u -d github.com/pressly/goose/cmd/goose
WORKDIR /go/src/github.com/pressly/goose/cmd/goose
RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -tags='no_mysql no_sqlite' -o goose .
WORKDIR /go/src/github.com/vulcanize/ipld-eth-server
ARG GOOSE_VER="v2.6.0"
ADD https://github.com/pressly/goose/releases/download/${GOOSE_VER}/goose-linux64 ./goose
RUN chmod +x ./goose
# app container
FROM alpine
ARG USER
ARG CONFIG_FILE
ARG EXPOSE_PORT_1
ARG EXPOSE_PORT_2
ARG USER="vdm"
ARG CONFIG_FILE="./environments/example.toml"
RUN adduser -Du 5000 $USER
WORKDIR /app
@ -33,17 +29,13 @@ USER $USER
# chown first so dir is writable
# note: using $USER is merged, but not in the stable release yet
COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/ipld-eth-server/$CONFIG_FILE config.toml
COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/ipld-eth-server/dockerfiles/super_node/startup_script.sh .
COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/ipld-eth-server/dockerfiles/super_node/entrypoint.sh .
COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/ipld-eth-server/entrypoint.sh .
# keep binaries immutable
COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-server/ipld-eth-server ipld-eth-server
COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose goose
COPY --from=builder /goose goose
COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-server/db/migrations migrations/vulcanizedb
COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-server/environments environments
EXPOSE $EXPOSE_PORT_1
EXPOSE $EXPOSE_PORT_2
ENTRYPOINT ["/app/startup_script.sh"]
ENTRYPOINT ["/app/entrypoint.sh"]

5
Makefile
View File

@ -137,3 +137,8 @@ version_migrations:
import:
test -n "$(NAME)" # $$NAME
psql $(NAME) < db/schema.sql
## Build docker image
.PHONY: docker-build
docker-build:
docker build -t vulcanize/ipld-eth-server .

78
docker-compose.yml Normal file
View File

@ -0,0 +1,78 @@
version: '3.2'
services:
dapptools:
restart: unless-stopped
image: vulcanize/dapptools:v0.29.0-statediff-0.0.2
ports:
- "127.0.0.1:8545:8545"
- "127.0.0.1:8546:8546"
db:
restart: always
image: postgres:10.12-alpine
environment:
POSTGRES_USER: "vdbm"
POSTGRES_DB: "vulcanize_public"
POSTGRES_PASSWORD: "password"
volumes:
- vdb_db_eth_server:/var/lib/postgresql/data
ports:
- "127.0.0.1:8077:5432"
eth-indexer:
restart: unless-stopped
depends_on:
- db
- dapptools
image: vulcanize/ipld-eth-indexer:v0.3.0-alpha
environment:
DATABASE_NAME: vulcanize_public
DATABASE_HOSTNAME: db
DATABASE_PORT: 5432
DATABASE_USER: vdbm
DATABASE_PASSWORD: password
ETH_WS_PATH: "dapptools:8546"
ETH_HTTP_PATH: "dapptools:8545"
ETH_CHAIN_ID: 4
ETH_NETWORK_ID: 4
VDB_COMMAND: sync
eth-server:
depends_on:
- db
build:
context: ./
cache_from:
- alpine:latest
- golang:1.13-alpine
environment:
VDB_COMMAND: "serve"
DATABASE_NAME: "vulcanize_public"
DATABASE_HOSTNAME: "db"
DATABASE_PORT: 5432
DATABASE_USER: "vdbm"
DATABASE_PASSWORD: "password"
SERVER_WS_PATH: "0.0.0.0:8081"
SERVER_HTTP_PATH: "0.0.0.0:8082"
ports:
- "127.0.0.1:8080:8080"
- "127.0.0.1:8081:8081"
graphql:
restart: unless-stopped
depends_on:
- db
image: vulcanize/postgraphile:v1.0.1
environment:
- PG_HOST=db
- PG_PORT=5432
- PG_DATABASE=vulcanize_public
- PG_USER=vdbm
- PG_PASSWORD=password
- SCHEMA=public,eth
ports:
- "127.0.0.1:5000:5000"
volumes:
vdb_db_eth_server:

40
dockerfiles/migrations/Dockerfile
View File

@ -1,40 +0,0 @@
FROM golang:alpine
RUN apk --update --no-cache add make git g++ linux-headers
# DEBUG
RUN apk add busybox-extras
# this is probably a noob move, but I want apk from alpine for the above but need to avoid Go 1.13 below as this error still occurs https://github.com/ipfs/go-ipfs/issues/6603
FROM golang:1.12.4 as builder
# Get and build ipld-eth-server
ADD . /go/src/github.com/vulcanize/ipld-eth-server
# Build migration tool
RUN go get -u -d github.com/pressly/goose/cmd/goose
WORKDIR /go/src/github.com/pressly/goose/cmd/goose
RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -tags='no_mysql no_sqlite' -o goose .
WORKDIR /go/src/github.com/vulcanize/ipld-eth-server
# app container
FROM alpine
ARG USER
RUN adduser -Du 5000 $USER
WORKDIR /app
RUN chown $USER /app
USER $USER
# chown first so dir is writable
# note: using $USER is merged, but not in the stable release yet
COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/ipld-eth-server/dockerfiles/migrations/startup_script.sh .
# keep binaries immutable
COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose goose
COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-server/db/migrations migrations/vulcanizedb
# XXX dir is already writeable RUN touch vulcanizedb.log
CMD ["./startup_script.sh"]

8
dockerfiles/postgraphile/Dockerfile
View File

@ -1,8 +0,0 @@
FROM node:alpine
RUN npm install -g postgraphile
RUN npm install -g postgraphile-plugin-connection-filter
RUN npm install -g @graphile/pg-pubsub
EXPOSE 5000
ENTRYPOINT ["postgraphile"]

60
dockerfiles/postgraphile/docker-compose.yml
View File

@ -1,60 +0,0 @@
version: '3.2'
services:
db:
restart: always
image: postgres:10.12-alpine
environment:
POSTGRES_USER: "vdbm"
POSTGRES_DB: "vulcanize_public"
POSTGRES_PASSWORD: "password"
volumes:
- vulcanizedb_db_data:/var/lib/postgresql/data
expose:
- "5432"
ports:
- "127.0.0.1:8079:5432"
migrations:
restart: on-failure
depends_on:
- db
build:
context: ./../../
cache_from:
- alpine:latest
dockerfile: ./dockerfiles/migrations/Dockerfile
args:
USER: "vdbm"
environment:
DATABASE_NAME: "vulcanize_public"
DATABASE_HOSTNAME: "db"
DATABASE_PORT: 5432
DATABASE_USER: "vdbm"
DATABASE_PASSWORD: "password"
graphql:
restart: always
depends_on:
- db
- migrations
build:
context: ./../../
cache_from:
- node:alpine
dockerfile: ./dockerfiles/postgraphile/Dockerfile
expose:
- "5000"
ports:
- "127.0.0.1:5000:5000"
command: ["--plugins", "@graphile/pg-pubsub",
"--subscriptions",
"--simple-subscriptions",
"--connection", "postgres://vdbm:password@db:5432/vulcanize_public",
"--port", "5000",
"-n", "0.0.0.0",
"--schema", "public,btc,eth",
"--append-plugins", "postgraphile-plugin-connection-filter"]
volumes:
vulcanizedb_db_data:

87
dockerfiles/super_node/docker-compose.yml
View File

@ -1,87 +0,0 @@
version: '3.2'
services:
db:
restart: always
image: postgres:10.12-alpine
environment:
POSTGRES_USER: "vdbm"
POSTGRES_DB: "vulcanize_public"
POSTGRES_PASSWORD: "password"
volumes:
- vulcanizedb_db_data:/var/lib/postgresql/data
expose:
- "5432"
ports:
- "127.0.0.1:8079:5432"
btc:
depends_on:
- db
build:
context: ./../../
cache_from:
- alpine:latest
- golang:1.12.4
dockerfile: ./dockerfiles/super_node/Dockerfile
args:
USER: "vdbm"
CONFIG_FILE: ./environments/superNodeBTC.toml
environment:
VDB_COMMAND: "watcher"
DATABASE_NAME: "vulcanize_public"
DATABASE_HOSTNAME: "db"
DATABASE_PORT: 5432
DATABASE_USER: "vdbm"
DATABASE_PASSWORD: "password"
ports:
- "127.0.0.1:8082:8082"
- "127.0.0.1:8083:8083"
eth:
depends_on:
- db
build:
context: ./../../
cache_from:
- alpine:latest
- golang:1.12.4
dockerfile: ./dockerfiles/super_node/Dockerfile
args:
USER: "vdbm"
CONFIG_FILE: ./environments/superNodeETH.toml
environment:
VDB_COMMAND: "watch"
DATABASE_NAME: "vulcanize_public"
DATABASE_HOSTNAME: "db"
DATABASE_PORT: 5432
DATABASE_USER: "vdbm"
DATABASE_PASSWORD: "password"
ports:
- "127.0.0.1:8080:8080"
- "127.0.0.1:8081:8081"
graphql:
restart: always
depends_on:
- db
build:
context: ./../../
cache_from:
- node:alpine
dockerfile: ./dockerfiles/postgraphile/Dockerfile
expose:
- "5000"
ports:
- "127.0.0.1:5000:5000"
command: ["--plugins", "@graphile/pg-pubsub",
"--subscriptions",
"--simple-subscriptions",
"--connection", "postgres://vdbm:password@db:5432/vulcanize_public",
"--port", "5000",
"-n", "0.0.0.0",
"--schema", "public,btc,eth",
"--append-plugins", "postgraphile-plugin-connection-filter"]
volumes:
vulcanizedb_db_data:

59
dockerfiles/super_node/entrypoint.sh
View File

@ -1,59 +0,0 @@
#!/bin/sh
# Runs the db migrations and starts the watcher services
# Exit if the variable tests fail
set -e
set +x
# Check the database variables are set
# XXX set defaults, don't silently fail
#test $DATABASE_HOSTNAME
#test $DATABASE_NAME
#test $DATABASE_PORT
#test $DATABASE_USER
#test $DATABASE_PASSWORD
#test $IPFS_INIT
VDB_COMMAND=${VDB_COMMAND:-watch}
set +e
# Construct the connection string for postgres
VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable
# Run the DB migrations
echo "Connecting with: $VDB_PG_CONNECT"
echo "Running database migrations"
./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up
rv=$?
if [ $rv != 0 ]; then
echo "Could not run migrations. Are the database details correct?"
exit 1
fi
echo "Beginning the vulcanizedb process"
VDB_CONFIG_FILE=${VDB_CONFIG_FILE:-config.toml}
DEFAULT_OPTIONS="--config=$VDB_CONFIG_FILE"
VDB_FULL_CL=${VDB_FULL_CL:-$VDB_COMMAND $DEFAULT_OPTIONS}
echo running: ./ipld-eth-server $VDB_FULL_CL $@
case "$1" in
"/bin/sh" )
echo dropping to shell
exec /bin/sh
esac
vdb_args="$@"
# default is to use the config passed by the build arg
if [[ -z "$vdb_args" ]]; then
vdb_args="--config=config.toml"
fi
echo running: ./ipld-eth-server $vdb_args
./ipld-eth-server $vdb_args
rv=$?
if [ $rv != 0 ]; then
echo "VulcanizeDB startup failed"
exit 1
fi

64
dockerfiles/super_node/startup_script.sh
View File

@ -1,64 +0,0 @@
#!/bin/sh
# Runs the db migrations and starts the watcher services
# Exit if the variable tests fail
set -e
set +x
# Check the database variables are set
test $DATABASE_HOSTNAME
test $DATABASE_NAME
test $DATABASE_PORT
test $DATABASE_USER
test $DATABASE_PASSWORD
test $IPFS_INIT
test $VDB_COMMAND
set +e
# Export our database variables so that the IPFS Postgres plugin can use them
export IPFS_PGHOST=$DATABASE_HOSTNAME
export IPFS_PGUSER=$DATABASE_USER
export IPFS_PGDATABASE=$DATABASE_NAME
export IPFS_PGPORT=$DATABASE_PORT
export IPFS_PGPASSWORD=$DATABASE_PASSWORD
# Construct the connection string for postgres
VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable
# Run the DB migrations
echo "Connecting with: $VDB_PG_CONNECT"
echo "Running database migrations"
./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up
# If the db migrations ran without err
if [[ $? -eq 0 ]]; then
# and IPFS_INIT is true
if [[ "$IPFS_INIT" = true ]] ; then
# initialize PG-IPFS
echo "Initializing Postgres-IPFS profile"
./ipfs init --profile=postgresds
else
echo "IPFS profile already initialized, skipping initialization"
fi
else
echo "Could not run migrations. Are the database details correct?"
exit 1
fi
# If IPFS initialization was successful
if [[ $? -eq 0 ]]; then
echo "Running the VulcanizeDB process"
./ipld-eth-server ${VDB_COMMAND} --config=config.toml
else
echo "Could not initialize IPFS."
exit 1
fi
# If VulcanizeDB process was successful
if [ $? -eq 0 ]; then
echo "VulcanizeDB process ran successfully"
else
echo "Could not start VulcanizeDB process. Is the config file correct?"
exit 1
fi

34
dockerfiles/migrations/startup_script.sh → entrypoint.sh
View File

@ -1,18 +1,6 @@
#!/bin/sh
# Runs the db migrations and starts the watcher services
# Exit if the variable tests fail
set -e
set +x
# Check the database variables are set
test $DATABASE_HOSTNAME
test $DATABASE_NAME
test $DATABASE_PORT
test $DATABASE_USER
test $DATABASE_PASSWORD
set +e
# Construct the connection string for postgres
VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable
@ -20,13 +8,21 @@ VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME
echo "Connecting with: $VDB_PG_CONNECT"
echo "Running database migrations"
./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up
rv=$?
i-norden commented 2020-09-09 18:37:35 +00:00
Review
Copy Link

I think we should get rid of this set +e and the set -e on line 5, and get rid of the commented out stuff in-between them.

I think we should get rid of this `set +e` and the `set -e` on line 5, and get rid of the commented out stuff in-between them.
if [ $rv != 0 ]; then
echo "Could not run migrations. Are the database details correct?"
exit 1
fi
# If the db migrations ran without err
if [[ $? -eq 0 ]]; then
echo "Migrations ran successfully"
exit 0
else
echo "Could not run migrations. Are the database details correct?"
exit 1
echo "Beginning the ipld-eth-server process"
echo running: ./ipld-eth-server ${VDB_COMMAND} --config=config.toml
./ipld-eth-server ${VDB_COMMAND} --config=config.toml
rv=$?
if [ $rv != 0 ]; then
echo "ipld-eth-server startup failed"
exit 1
fi

6
pkg/serve/config.go
View File

@ -60,12 +60,12 @@ func NewConfig() (*Config, error) {
c.DBConfig.Init()
wsPath := viper.GetString("watcher.wsPath")
wsPath := viper.GetString("server.wsPath")
if wsPath == "" {
i-norden commented 2020-09-09 18:38:08 +00:00
Review
Copy Link

Nice catch, thanks for fixing these!

Nice catch, thanks for fixing these!
wsPath = "127.0.0.1:8080"
}
c.WSEndpoint = wsPath
ipcPath := viper.GetString("watcher.ipcPath")
ipcPath := viper.GetString("server.ipcPath")
if ipcPath == "" {
home, err := os.UserHomeDir()
if err != nil {
@ -74,7 +74,7 @@ func NewConfig() (*Config, error) {
ipcPath = filepath.Join(home, ".vulcanize/vulcanize.ipc")
}
c.IPCEndpoint = ipcPath
httpPath := viper.GetString("watcher.httpPath")
httpPath := viper.GetString("server.httpPath")
if httpPath == "" {
httpPath = "127.0.0.1:8081"
}