From 5dec3d145d07d2c93c68d9dc5475c6cabb9df11e Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Mon, 27 Jan 2020 14:13:54 -0600 Subject: [PATCH] updating SuperNode dockerfile --- dockerfiles/super_node/Dockerfile | 32 +++++------------------- dockerfiles/super_node/startup_script.sh | 30 ++++++++++------------ documentation/super_node/setup.md | 9 +++---- environments/superNode.toml | 4 +-- 4 files changed, 25 insertions(+), 50 deletions(-) diff --git a/dockerfiles/super_node/Dockerfile b/dockerfiles/super_node/Dockerfile index 82f6cb41..a39f883c 100644 --- a/dockerfiles/super_node/Dockerfile +++ b/dockerfiles/super_node/Dockerfile @@ -7,10 +7,11 @@ RUN apk add busybox-extras # this is probably a noob move, but I want apk from alpine for the above but need to avoid Go 1.13 below as this error still occurs https://github.com/ipfs/go-ipfs/issues/6603 FROM golang:1.12.4 as builder -# Get and build vulcanizedb ipfs_concurreny fork -RUN go get -u -d github.com/vulcanize/vulcanizedb +RUN yum install -y libusb1-devel systemd-devel + +# Get and build vulcanizedb +ADD . /go/src/github.com/vulcanize/vulcanizedb WORKDIR /go/src/github.com/vulcanize/vulcanizedb -RUN git checkout ipfs_concurrency RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o vulcanizedb . # Get and build vulcanize's go-ipfs fork @@ -18,17 +19,9 @@ RUN go get -u -d github.com/ipfs/go-ipfs WORKDIR /go/src/github.com/ipfs/go-ipfs RUN git remote add vulcanize https://github.com/vulcanize/go-ipfs.git RUN git fetch vulcanize -RUN git checkout -b pg_ipfs vulcanize/postgres_update +RUN git checkout -b pg_ipfs v0.4.22-alpha RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o ipfs ./cmd/ipfs -# Get and build vulcanize's geth fork -RUN go get -u -d github.com/ethereum/go-ethereum -WORKDIR /go/src/github.com/ethereum/go-ethereum -RUN git remote add vulcanize https://github.com/vulcanize/go-ethereum.git -RUN git fetch vulcanize -RUN git checkout -b statediff_geth vulcanize/statediffing -RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o geth ./cmd/geth - # Build migration tool RUN go get -u -d github.com/pressly/goose/cmd/goose WORKDIR /go/src/github.com/pressly/goose/cmd/goose @@ -41,19 +34,7 @@ FROM alpine WORKDIR /app ARG USER -ARG config_file=environments/syncPublishScreenAndServe.toml -ARG vdb_dbname="vulcanize_public" -ARG vdb_hostname="localhost" -ARG vdb_port="5432" -ARG vdb_user="postgres" -ARG vdb_password - -# setup environment -ENV VDB_PG_NAME="$vdb_dbname" -ENV VDB_PG_HOSTNAME="$vdb_hostname" -ENV VDB_PG_PORT="$vdb_port" -ENV VDB_PG_USER="$vdb_user" -ENV VDB_PG_PASSWORD="$vdb_password" +ARG config_file=environments/superNode.toml RUN adduser -D 5000 $USER USER $USER @@ -68,7 +49,6 @@ COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/vulcanizedb vulcani COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose goose COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/db/migrations migrations/vulcanizedb COPY --from=builder /go/src/github.com/ipfs/go-ipfs/ipfs ipfs -COPY --from=builder /go/src/github.com/ethereum/go-ethereum/geth geth EXPOSE 8080 diff --git a/dockerfiles/super_node/startup_script.sh b/dockerfiles/super_node/startup_script.sh index 203035c0..ddf6c8a1 100755 --- a/dockerfiles/super_node/startup_script.sh +++ b/dockerfiles/super_node/startup_script.sh @@ -9,6 +9,7 @@ test $VDB_PG_NAME test $VDB_PG_HOSTNAME test $VDB_PG_PORT test $VDB_PG_USER +test $IPFS_INIT set +e # Export our database variables so that the IPFS Postgres plugin can use them @@ -26,35 +27,30 @@ echo "Connecting with: $VDB_PG_CONNECT" echo "Running database migrations" ./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up + # If the db migrations ran without err -if [ $? -eq 0 ]; then - # Initialize PG-IPFS - echo "Initializing Postgres-IPFS profile" - ./ipfs init --profile=postgresds +if [[ $? -eq 0 ]]; then + # and IPFS_INIT is true + if [[ "$IPFS_INIT" = true ]] ; then + # initialize PG-IPFS + echo "Initializing Postgres-IPFS profile" + ./ipfs init --profile=postgresds + else + echo "IPFS profile already initialized, skipping initialization" + fi else echo "Could not run migrations. Are the database details correct?" exit fi # If IPFS initialization was successful -if [ $? -eq 0 ]; then - # Begin the state-diffing Geth process - echo "Beginning the state-diffing Geth process" - ./geth --statediff --statediff.streamblock --ws --syncmode=full 2>&1 | tee -a log.txt & - sleep 1 -else - echo "Could not initialize Postgres backed IPFS profile. Are the database details correct?" - exit -fi - -# If Geth startup was successful -if [ $? -eq 0 ]; then +if [[ $? -eq 0 ]]; then # Wait until block synchronisation has begun echo "Waiting for block synchronization to begin" ( tail -f -n0 log.txt & ) | grep -q "Block synchronisation started" # this blocks til we see "Block synchronisation started" # And then spin up the syncPublishScreenAndServe Vulcanizedb service echo "Beginning the syncPublishScreenAndServe vulcanizedb process" - ./vulcanizedb syncPublishScreenAndServe --config=config.toml 2>&1 | tee -a log.txt & + ./vulcanizedb superNode --config=config.toml 2>&1 | tee -a log.txt & else echo "Could not initialize state-diffing Geth." exit diff --git a/documentation/super_node/setup.md b/documentation/super_node/setup.md index d3f20a28..314ab8a2 100644 --- a/documentation/super_node/setup.md +++ b/documentation/super_node/setup.md @@ -171,11 +171,11 @@ sudo -u postgres createdb ec2-user sudo su postgres psql ALTER USER "ec2-user" WITH SUPERUSER; -/q +\q exit ``` -4b. Edit hba_file to trust connections +4b. Edit hba_file to trust local connections ``` psql SHOW hba_file; @@ -196,11 +196,10 @@ sudo service docker start sudo usermod -aG docker ec2-user ``` -6. Fetch the repository and switch to this working branch +6. Fetch the repository ``` go get github.com/vulcanize/vulcanizedb cd $GOPATH/src/github.com/vulcanize/vulcanizedb -git checkout ipfs_concurrency ``` 7. Create the db @@ -212,5 +211,5 @@ createdb vulcanize_public ``` cd $GOPATH/src/github.com/vulcanize/vulcanizedb/dockerfiles/super_node docker build . -docker run --network host -e VDB_PG_CONNECT=postgres://localhost:5432/vulcanize_public?sslmode=disable {IMAGE_ID} +docker run --network host -e IPFS_INIT=true -e VDB_PG_NAME=vulcanize_public -e VDB_PG_HOSTNAME=localhost -e VDB_PG_PORT=5432 -e VDB_PG_USER=postgres -e VDB_PG_PASSWORD=password {IMAGE_ID} ``` \ No newline at end of file diff --git a/environments/superNode.toml b/environments/superNode.toml index 5447c365..39391864 100644 --- a/environments/superNode.toml +++ b/environments/superNode.toml @@ -19,7 +19,7 @@ wsPath = "127.0.0.1:8080" [superNode.backFill] - on = false - httpPath = "" + on = true + httpPath = "http://127.0.0.1:8545" frequency = 5 batchSize = 50 \ No newline at end of file