updating SuperNode dockerfile

dockerfiles/super_node/Dockerfile

@@ -7,10 +7,11 @@ RUN apk add busybox-extras
 # this is probably a noob move, but I want apk from alpine for the above but need to avoid Go 1.13 below as this error still occurs https://github.com/ipfs/go-ipfs/issues/6603
 FROM golang:1.12.4 as builder
 
-# Get and build vulcanizedb ipfs_concurreny fork
+RUN yum install -y libusb1-devel systemd-devel
-RUN go get -u -d github.com/vulcanize/vulcanizedb
+
+# Get and build vulcanizedb
+ADD . /go/src/github.com/vulcanize/vulcanizedb
 WORKDIR /go/src/github.com/vulcanize/vulcanizedb
-RUN git checkout ipfs_concurrency
 RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o vulcanizedb .
 
 # Get and build vulcanize's go-ipfs fork
@@ -18,17 +19,9 @@ RUN go get -u -d github.com/ipfs/go-ipfs
 WORKDIR /go/src/github.com/ipfs/go-ipfs
 RUN git remote add vulcanize https://github.com/vulcanize/go-ipfs.git
 RUN git fetch vulcanize
-RUN git checkout -b pg_ipfs vulcanize/postgres_update
+RUN git checkout -b pg_ipfs v0.4.22-alpha
 RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o ipfs ./cmd/ipfs
 
-# Get and build vulcanize's geth fork
-RUN go get -u -d github.com/ethereum/go-ethereum
-WORKDIR /go/src/github.com/ethereum/go-ethereum
-RUN git remote add vulcanize https://github.com/vulcanize/go-ethereum.git
-RUN git fetch vulcanize
-RUN git checkout -b statediff_geth vulcanize/statediffing
-RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o geth ./cmd/geth
-
 # Build migration tool
 RUN go get -u -d github.com/pressly/goose/cmd/goose
 WORKDIR /go/src/github.com/pressly/goose/cmd/goose
@@ -41,19 +34,7 @@ FROM alpine
 WORKDIR /app
 
 ARG USER
-ARG config_file=environments/syncPublishScreenAndServe.toml
+ARG config_file=environments/superNode.toml
-ARG vdb_dbname="vulcanize_public"
-ARG vdb_hostname="localhost"
-ARG vdb_port="5432"
-ARG vdb_user="postgres"
-ARG vdb_password
-
-# setup environment
-ENV VDB_PG_NAME="$vdb_dbname"
-ENV VDB_PG_HOSTNAME="$vdb_hostname"
-ENV VDB_PG_PORT="$vdb_port"
-ENV VDB_PG_USER="$vdb_user"
-ENV VDB_PG_PASSWORD="$vdb_password"
 
 RUN adduser -D 5000 $USER
 USER $USER
@@ -68,7 +49,6 @@ COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/vulcanizedb vulcani
 COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose goose
 COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/db/migrations migrations/vulcanizedb
 COPY --from=builder /go/src/github.com/ipfs/go-ipfs/ipfs ipfs
-COPY --from=builder /go/src/github.com/ethereum/go-ethereum/geth geth
 
 EXPOSE 8080
 

dockerfiles/super_node/startup_script.sh

@@ -9,6 +9,7 @@ test $VDB_PG_NAME
 test $VDB_PG_HOSTNAME
 test $VDB_PG_PORT
 test $VDB_PG_USER
+test $IPFS_INIT
 set +e
 
 # Export our database variables so that the IPFS Postgres plugin can use them
@@ -26,35 +27,30 @@ echo "Connecting with: $VDB_PG_CONNECT"
 echo "Running database migrations"
 ./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up
 
+
 # If the db migrations ran without err
-if [ $? -eq 0 ]; then
+if [[ $? -eq 0 ]]; then
-    # Initialize PG-IPFS
+    # and IPFS_INIT is true
+    if [[ "$IPFS_INIT" = true ]] ; then
+        # initialize PG-IPFS
         echo "Initializing Postgres-IPFS profile"
         ./ipfs init --profile=postgresds
+    else
+        echo "IPFS profile already initialized, skipping initialization"
+    fi
 else
     echo "Could not run migrations. Are the database details correct?"
     exit
 fi
 
 # If IPFS initialization was successful
-if [ $? -eq 0 ]; then
+if [[ $? -eq 0 ]]; then
-    # Begin the state-diffing Geth process
-    echo "Beginning the state-diffing Geth process"
-    ./geth --statediff --statediff.streamblock --ws --syncmode=full 2>&1 | tee -a log.txt &
-    sleep 1
-else
-    echo "Could not initialize Postgres backed IPFS profile. Are the database details correct?"
-    exit
-fi
-
-# If Geth startup was successful
-if [ $? -eq 0 ]; then
     # Wait until block synchronisation has begun
     echo "Waiting for block synchronization to begin"
     ( tail -f -n0 log.txt & ) | grep -q "Block synchronisation started" # this blocks til we see "Block synchronisation started"
     # And then spin up the syncPublishScreenAndServe Vulcanizedb service
     echo "Beginning the syncPublishScreenAndServe vulcanizedb process"
-    ./vulcanizedb syncPublishScreenAndServe --config=config.toml 2>&1 | tee -a log.txt &
+    ./vulcanizedb superNode --config=config.toml 2>&1 | tee -a log.txt &
 else
     echo "Could not initialize state-diffing Geth."
     exit

documentation/super_node/setup.md

@@ -171,11 +171,11 @@ sudo -u postgres createdb ec2-user
 sudo su postgres
 psql
 ALTER USER "ec2-user" WITH SUPERUSER;
-/q
+\q
 exit
 ```
 
-4b. Edit hba_file to trust connections
+4b. Edit hba_file to trust local connections
 ```
 psql
 SHOW hba_file;
@@ -196,11 +196,10 @@ sudo service  docker start
 sudo usermod -aG docker ec2-user
 ```
 
-6. Fetch the repository and switch to this working branch
+6. Fetch the repository
 ```
 go get github.com/vulcanize/vulcanizedb
 cd $GOPATH/src/github.com/vulcanize/vulcanizedb
-git checkout ipfs_concurrency
 ```
 
 7. Create the db
@@ -212,5 +211,5 @@ createdb vulcanize_public
 ```
 cd $GOPATH/src/github.com/vulcanize/vulcanizedb/dockerfiles/super_node
 docker build .
-docker run --network host -e VDB_PG_CONNECT=postgres://localhost:5432/vulcanize_public?sslmode=disable {IMAGE_ID}
+docker run --network host -e IPFS_INIT=true -e VDB_PG_NAME=vulcanize_public -e VDB_PG_HOSTNAME=localhost -e VDB_PG_PORT=5432 -e VDB_PG_USER=postgres -e VDB_PG_PASSWORD=password {IMAGE_ID}
 ```

environments/superNode.toml

@@ -19,7 +19,7 @@
         wsPath = "127.0.0.1:8080"
 
     [superNode.backFill]
-        on = false
+        on = true
-        httpPath = ""
+        httpPath = "http://127.0.0.1:8545"
         frequency = 5
         batchSize = 50