diff --git a/.github/workflows/on-master.yaml b/.github/workflows/on-master.yaml new file mode 100644 index 00000000..aa7c66e6 --- /dev/null +++ b/.github/workflows/on-master.yaml @@ -0,0 +1,25 @@ +name: Docker Compose Build + +on: + push: + branches: + - master + +jobs: + build: + name: Run docker build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Get the version + id: vars + run: echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7}) + - name: Run docker build + run: make docker-build + - name: Tag docker image + run: docker tag vulcanize/ipld-eth-server docker.pkg.github.com/vulcanize/ipld-eth-server/ipld-eth-server:${{steps.vars.outputs.sha}} + - name: Docker Login + run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin + - name: Docker Push + run: docker push docker.pkg.github.com/vulcanize/ipld-eth-server/ipld-eth-server:${{steps.vars.outputs.sha}} + diff --git a/.github/workflows/on-pr.yaml b/.github/workflows/on-pr.yaml new file mode 100644 index 00000000..568dd104 --- /dev/null +++ b/.github/workflows/on-pr.yaml @@ -0,0 +1,12 @@ +name: Docker Build + +on: [pull_request] + +jobs: + build: + name: Run docker build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Run docker build + run: make docker-build diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml new file mode 100644 index 00000000..42450ce7 --- /dev/null +++ b/.github/workflows/publish.yaml @@ -0,0 +1,25 @@ +name: Publish Docker image +on: + release: + types: [published] +jobs: + push_to_registries: + name: Push Docker image to Docker Hub + runs-on: ubuntu-latest + steps: + - name: Get the version + id: vars + run: | + echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7}) + echo ::set-output name=tag::$(echo ${GITHUB_REF#refs/tags/}) + - name: Docker Login to Github Registry + run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin + - name: Docker Pull + run: docker pull docker.pkg.github.com/vulcanize/ipld-eth-server/ipld-eth-server:${{steps.vars.outputs.sha}} + - name: Docker Login to Docker Registry + run: echo ${{ secrets.VULCANIZEJENKINS_PAT }} | docker login -u vulcanizejenkins --password-stdin + - name: Tag docker image + run: docker tag docker.pkg.github.com/vulcanize/ipld-eth-server/ipld-eth-server:${{steps.vars.outputs.sha}} vulcanize/ipld-eth-server:${{steps.vars.outputs.tag}} + - name: Docker Push to Docker Hub + run: docker push vulcanize/ipld-eth-server:${{steps.vars.outputs.tag}} + diff --git a/dockerfiles/super_node/Dockerfile b/Dockerfile similarity index 56% rename from dockerfiles/super_node/Dockerfile rename to Dockerfile index df9c6f1e..a7c72db7 100644 --- a/dockerfiles/super_node/Dockerfile +++ b/Dockerfile @@ -4,26 +4,22 @@ RUN apk --update --no-cache add make git g++ linux-headers # DEBUG RUN apk add busybox-extras -# Get and build ipld-eth-server -ADD . /go/src/github.com/vulcanize/ipld-eth-server +# Build ipld-eth-server WORKDIR /go/src/github.com/vulcanize/ipld-eth-server +ADD . . RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o ipld-eth-server . -# Build migration tool +# Copy migration tool WORKDIR / -RUN go get -u -d github.com/pressly/goose/cmd/goose -WORKDIR /go/src/github.com/pressly/goose/cmd/goose -RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -tags='no_mysql no_sqlite' -o goose . - -WORKDIR /go/src/github.com/vulcanize/ipld-eth-server +ARG GOOSE_VER="v2.6.0" +ADD https://github.com/pressly/goose/releases/download/${GOOSE_VER}/goose-linux64 ./goose +RUN chmod +x ./goose # app container FROM alpine -ARG USER -ARG CONFIG_FILE -ARG EXPOSE_PORT_1 -ARG EXPOSE_PORT_2 +ARG USER="vdm" +ARG CONFIG_FILE="./environments/example.toml" RUN adduser -Du 5000 $USER WORKDIR /app @@ -33,17 +29,13 @@ USER $USER # chown first so dir is writable # note: using $USER is merged, but not in the stable release yet COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/ipld-eth-server/$CONFIG_FILE config.toml -COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/ipld-eth-server/dockerfiles/super_node/startup_script.sh . -COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/ipld-eth-server/dockerfiles/super_node/entrypoint.sh . +COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/ipld-eth-server/entrypoint.sh . # keep binaries immutable COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-server/ipld-eth-server ipld-eth-server -COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose goose +COPY --from=builder /goose goose COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-server/db/migrations migrations/vulcanizedb COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-server/environments environments -EXPOSE $EXPOSE_PORT_1 -EXPOSE $EXPOSE_PORT_2 - -ENTRYPOINT ["/app/startup_script.sh"] +ENTRYPOINT ["/app/entrypoint.sh"] diff --git a/Makefile b/Makefile index 7b1a680c..e4a64153 100644 --- a/Makefile +++ b/Makefile @@ -137,3 +137,8 @@ version_migrations: import: test -n "$(NAME)" # $$NAME psql $(NAME) < db/schema.sql + +## Build docker image +.PHONY: docker-build +docker-build: + docker build -t vulcanize/ipld-eth-server . \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..03942091 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,78 @@ +version: '3.2' + +services: + dapptools: + restart: unless-stopped + image: vulcanize/dapptools:v0.29.0-statediff-0.0.2 + ports: + - "127.0.0.1:8545:8545" + - "127.0.0.1:8546:8546" + + db: + restart: always + image: postgres:10.12-alpine + environment: + POSTGRES_USER: "vdbm" + POSTGRES_DB: "vulcanize_public" + POSTGRES_PASSWORD: "password" + volumes: + - vdb_db_eth_server:/var/lib/postgresql/data + ports: + - "127.0.0.1:8077:5432" + + eth-indexer: + restart: unless-stopped + depends_on: + - db + - dapptools + image: vulcanize/ipld-eth-indexer:v0.3.0-alpha + environment: + DATABASE_NAME: vulcanize_public + DATABASE_HOSTNAME: db + DATABASE_PORT: 5432 + DATABASE_USER: vdbm + DATABASE_PASSWORD: password + ETH_WS_PATH: "dapptools:8546" + ETH_HTTP_PATH: "dapptools:8545" + ETH_CHAIN_ID: 4 + ETH_NETWORK_ID: 4 + VDB_COMMAND: sync + + eth-server: + depends_on: + - db + build: + context: ./ + cache_from: + - alpine:latest + - golang:1.13-alpine + environment: + VDB_COMMAND: "serve" + DATABASE_NAME: "vulcanize_public" + DATABASE_HOSTNAME: "db" + DATABASE_PORT: 5432 + DATABASE_USER: "vdbm" + DATABASE_PASSWORD: "password" + SERVER_WS_PATH: "0.0.0.0:8081" + SERVER_HTTP_PATH: "0.0.0.0:8082" + ports: + - "127.0.0.1:8080:8080" + - "127.0.0.1:8081:8081" + + graphql: + restart: unless-stopped + depends_on: + - db + image: vulcanize/postgraphile:v1.0.1 + environment: + - PG_HOST=db + - PG_PORT=5432 + - PG_DATABASE=vulcanize_public + - PG_USER=vdbm + - PG_PASSWORD=password + - SCHEMA=public,eth + ports: + - "127.0.0.1:5000:5000" + +volumes: + vdb_db_eth_server: diff --git a/dockerfiles/migrations/Dockerfile b/dockerfiles/migrations/Dockerfile deleted file mode 100644 index bce16c9d..00000000 --- a/dockerfiles/migrations/Dockerfile +++ /dev/null @@ -1,40 +0,0 @@ -FROM golang:alpine - -RUN apk --update --no-cache add make git g++ linux-headers -# DEBUG -RUN apk add busybox-extras - -# this is probably a noob move, but I want apk from alpine for the above but need to avoid Go 1.13 below as this error still occurs https://github.com/ipfs/go-ipfs/issues/6603 -FROM golang:1.12.4 as builder - -# Get and build ipld-eth-server -ADD . /go/src/github.com/vulcanize/ipld-eth-server - -# Build migration tool -RUN go get -u -d github.com/pressly/goose/cmd/goose -WORKDIR /go/src/github.com/pressly/goose/cmd/goose -RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -tags='no_mysql no_sqlite' -o goose . - -WORKDIR /go/src/github.com/vulcanize/ipld-eth-server - -# app container -FROM alpine - -ARG USER - -RUN adduser -Du 5000 $USER -WORKDIR /app -RUN chown $USER /app -USER $USER - -# chown first so dir is writable -# note: using $USER is merged, but not in the stable release yet -COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/ipld-eth-server/dockerfiles/migrations/startup_script.sh . - - -# keep binaries immutable -COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose goose -COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-server/db/migrations migrations/vulcanizedb -# XXX dir is already writeable RUN touch vulcanizedb.log - -CMD ["./startup_script.sh"] \ No newline at end of file diff --git a/dockerfiles/postgraphile/Dockerfile b/dockerfiles/postgraphile/Dockerfile deleted file mode 100644 index 8ce064bb..00000000 --- a/dockerfiles/postgraphile/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM node:alpine - -RUN npm install -g postgraphile -RUN npm install -g postgraphile-plugin-connection-filter -RUN npm install -g @graphile/pg-pubsub - -EXPOSE 5000 -ENTRYPOINT ["postgraphile"] \ No newline at end of file diff --git a/dockerfiles/postgraphile/docker-compose.yml b/dockerfiles/postgraphile/docker-compose.yml deleted file mode 100644 index 5ee8059e..00000000 --- a/dockerfiles/postgraphile/docker-compose.yml +++ /dev/null @@ -1,60 +0,0 @@ -version: '3.2' - -services: - db: - restart: always - image: postgres:10.12-alpine - environment: - POSTGRES_USER: "vdbm" - POSTGRES_DB: "vulcanize_public" - POSTGRES_PASSWORD: "password" - volumes: - - vulcanizedb_db_data:/var/lib/postgresql/data - expose: - - "5432" - ports: - - "127.0.0.1:8079:5432" - - migrations: - restart: on-failure - depends_on: - - db - build: - context: ./../../ - cache_from: - - alpine:latest - dockerfile: ./dockerfiles/migrations/Dockerfile - args: - USER: "vdbm" - environment: - DATABASE_NAME: "vulcanize_public" - DATABASE_HOSTNAME: "db" - DATABASE_PORT: 5432 - DATABASE_USER: "vdbm" - DATABASE_PASSWORD: "password" - - graphql: - restart: always - depends_on: - - db - - migrations - build: - context: ./../../ - cache_from: - - node:alpine - dockerfile: ./dockerfiles/postgraphile/Dockerfile - expose: - - "5000" - ports: - - "127.0.0.1:5000:5000" - command: ["--plugins", "@graphile/pg-pubsub", - "--subscriptions", - "--simple-subscriptions", - "--connection", "postgres://vdbm:password@db:5432/vulcanize_public", - "--port", "5000", - "-n", "0.0.0.0", - "--schema", "public,btc,eth", - "--append-plugins", "postgraphile-plugin-connection-filter"] - -volumes: - vulcanizedb_db_data: \ No newline at end of file diff --git a/dockerfiles/super_node/docker-compose.yml b/dockerfiles/super_node/docker-compose.yml deleted file mode 100644 index c0b03e23..00000000 --- a/dockerfiles/super_node/docker-compose.yml +++ /dev/null @@ -1,87 +0,0 @@ -version: '3.2' - -services: - db: - restart: always - image: postgres:10.12-alpine - environment: - POSTGRES_USER: "vdbm" - POSTGRES_DB: "vulcanize_public" - POSTGRES_PASSWORD: "password" - volumes: - - vulcanizedb_db_data:/var/lib/postgresql/data - expose: - - "5432" - ports: - - "127.0.0.1:8079:5432" - - btc: - depends_on: - - db - build: - context: ./../../ - cache_from: - - alpine:latest - - golang:1.12.4 - dockerfile: ./dockerfiles/super_node/Dockerfile - args: - USER: "vdbm" - CONFIG_FILE: ./environments/superNodeBTC.toml - environment: - VDB_COMMAND: "watcher" - DATABASE_NAME: "vulcanize_public" - DATABASE_HOSTNAME: "db" - DATABASE_PORT: 5432 - DATABASE_USER: "vdbm" - DATABASE_PASSWORD: "password" - ports: - - "127.0.0.1:8082:8082" - - "127.0.0.1:8083:8083" - - eth: - depends_on: - - db - build: - context: ./../../ - cache_from: - - alpine:latest - - golang:1.12.4 - dockerfile: ./dockerfiles/super_node/Dockerfile - args: - USER: "vdbm" - CONFIG_FILE: ./environments/superNodeETH.toml - environment: - VDB_COMMAND: "watch" - DATABASE_NAME: "vulcanize_public" - DATABASE_HOSTNAME: "db" - DATABASE_PORT: 5432 - DATABASE_USER: "vdbm" - DATABASE_PASSWORD: "password" - ports: - - "127.0.0.1:8080:8080" - - "127.0.0.1:8081:8081" - - graphql: - restart: always - depends_on: - - db - build: - context: ./../../ - cache_from: - - node:alpine - dockerfile: ./dockerfiles/postgraphile/Dockerfile - expose: - - "5000" - ports: - - "127.0.0.1:5000:5000" - command: ["--plugins", "@graphile/pg-pubsub", - "--subscriptions", - "--simple-subscriptions", - "--connection", "postgres://vdbm:password@db:5432/vulcanize_public", - "--port", "5000", - "-n", "0.0.0.0", - "--schema", "public,btc,eth", - "--append-plugins", "postgraphile-plugin-connection-filter"] - -volumes: - vulcanizedb_db_data: diff --git a/dockerfiles/super_node/entrypoint.sh b/dockerfiles/super_node/entrypoint.sh deleted file mode 100755 index b203311a..00000000 --- a/dockerfiles/super_node/entrypoint.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/sh -# Runs the db migrations and starts the watcher services - -# Exit if the variable tests fail -set -e -set +x - -# Check the database variables are set -# XXX set defaults, don't silently fail -#test $DATABASE_HOSTNAME -#test $DATABASE_NAME -#test $DATABASE_PORT -#test $DATABASE_USER -#test $DATABASE_PASSWORD -#test $IPFS_INIT -VDB_COMMAND=${VDB_COMMAND:-watch} -set +e - -# Construct the connection string for postgres -VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable - -# Run the DB migrations -echo "Connecting with: $VDB_PG_CONNECT" -echo "Running database migrations" -./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up -rv=$? - -if [ $rv != 0 ]; then - echo "Could not run migrations. Are the database details correct?" - exit 1 -fi - - -echo "Beginning the vulcanizedb process" -VDB_CONFIG_FILE=${VDB_CONFIG_FILE:-config.toml} -DEFAULT_OPTIONS="--config=$VDB_CONFIG_FILE" -VDB_FULL_CL=${VDB_FULL_CL:-$VDB_COMMAND $DEFAULT_OPTIONS} -echo running: ./ipld-eth-server $VDB_FULL_CL $@ - -case "$1" in - "/bin/sh" ) - echo dropping to shell - exec /bin/sh -esac - -vdb_args="$@" -# default is to use the config passed by the build arg -if [[ -z "$vdb_args" ]]; then - vdb_args="--config=config.toml" -fi - -echo running: ./ipld-eth-server $vdb_args -./ipld-eth-server $vdb_args -rv=$? - -if [ $rv != 0 ]; then - echo "VulcanizeDB startup failed" - exit 1 -fi \ No newline at end of file diff --git a/dockerfiles/super_node/startup_script.sh b/dockerfiles/super_node/startup_script.sh deleted file mode 100755 index 326b1baf..00000000 --- a/dockerfiles/super_node/startup_script.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/sh -# Runs the db migrations and starts the watcher services - -# Exit if the variable tests fail -set -e -set +x - -# Check the database variables are set -test $DATABASE_HOSTNAME -test $DATABASE_NAME -test $DATABASE_PORT -test $DATABASE_USER -test $DATABASE_PASSWORD -test $IPFS_INIT -test $VDB_COMMAND -set +e - -# Export our database variables so that the IPFS Postgres plugin can use them -export IPFS_PGHOST=$DATABASE_HOSTNAME -export IPFS_PGUSER=$DATABASE_USER -export IPFS_PGDATABASE=$DATABASE_NAME -export IPFS_PGPORT=$DATABASE_PORT -export IPFS_PGPASSWORD=$DATABASE_PASSWORD - -# Construct the connection string for postgres -VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable - -# Run the DB migrations -echo "Connecting with: $VDB_PG_CONNECT" -echo "Running database migrations" -./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up - - -# If the db migrations ran without err -if [[ $? -eq 0 ]]; then - # and IPFS_INIT is true - if [[ "$IPFS_INIT" = true ]] ; then - # initialize PG-IPFS - echo "Initializing Postgres-IPFS profile" - ./ipfs init --profile=postgresds - else - echo "IPFS profile already initialized, skipping initialization" - fi -else - echo "Could not run migrations. Are the database details correct?" - exit 1 -fi - -# If IPFS initialization was successful -if [[ $? -eq 0 ]]; then - echo "Running the VulcanizeDB process" - ./ipld-eth-server ${VDB_COMMAND} --config=config.toml -else - echo "Could not initialize IPFS." - exit 1 -fi - -# If VulcanizeDB process was successful -if [ $? -eq 0 ]; then - echo "VulcanizeDB process ran successfully" -else - echo "Could not start VulcanizeDB process. Is the config file correct?" - exit 1 -fi \ No newline at end of file diff --git a/dockerfiles/migrations/startup_script.sh b/entrypoint.sh similarity index 50% rename from dockerfiles/migrations/startup_script.sh rename to entrypoint.sh index ddd45182..2c9e9821 100755 --- a/dockerfiles/migrations/startup_script.sh +++ b/entrypoint.sh @@ -1,18 +1,6 @@ #!/bin/sh # Runs the db migrations and starts the watcher services -# Exit if the variable tests fail -set -e -set +x - -# Check the database variables are set -test $DATABASE_HOSTNAME -test $DATABASE_NAME -test $DATABASE_PORT -test $DATABASE_USER -test $DATABASE_PASSWORD -set +e - # Construct the connection string for postgres VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable @@ -20,13 +8,21 @@ VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME echo "Connecting with: $VDB_PG_CONNECT" echo "Running database migrations" ./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up +rv=$? + +if [ $rv != 0 ]; then + echo "Could not run migrations. Are the database details correct?" + exit 1 +fi -# If the db migrations ran without err -if [[ $? -eq 0 ]]; then - echo "Migrations ran successfully" - exit 0 -else - echo "Could not run migrations. Are the database details correct?" - exit 1 +echo "Beginning the ipld-eth-server process" + +echo running: ./ipld-eth-server ${VDB_COMMAND} --config=config.toml +./ipld-eth-server ${VDB_COMMAND} --config=config.toml +rv=$? + +if [ $rv != 0 ]; then + echo "ipld-eth-server startup failed" + exit 1 fi \ No newline at end of file diff --git a/pkg/serve/config.go b/pkg/serve/config.go index 382d533f..821ee8dc 100644 --- a/pkg/serve/config.go +++ b/pkg/serve/config.go @@ -60,12 +60,12 @@ func NewConfig() (*Config, error) { c.DBConfig.Init() - wsPath := viper.GetString("watcher.wsPath") + wsPath := viper.GetString("server.wsPath") if wsPath == "" { wsPath = "127.0.0.1:8080" } c.WSEndpoint = wsPath - ipcPath := viper.GetString("watcher.ipcPath") + ipcPath := viper.GetString("server.ipcPath") if ipcPath == "" { home, err := os.UserHomeDir() if err != nil { @@ -74,7 +74,7 @@ func NewConfig() (*Config, error) { ipcPath = filepath.Join(home, ".vulcanize/vulcanize.ipc") } c.IPCEndpoint = ipcPath - httpPath := viper.GetString("watcher.httpPath") + httpPath := viper.GetString("server.httpPath") if httpPath == "" { httpPath = "127.0.0.1:8081" }