diff --git a/.gitignore b/.gitignore index 96f02e19..4cefb254 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ db/migrations/20*.sql plugins/*.so postgraphile/*.toml postgraphile/schema.graphql +vulcanizedb.pem diff --git a/cmd/streamSubscribe.go b/cmd/streamSubscribe.go index 4bece647..63059883 100644 --- a/cmd/streamSubscribe.go +++ b/cmd/streamSubscribe.go @@ -50,7 +50,6 @@ func init() { } func streamSubscribe() { - log.SetLevel(log.DebugLevel) // Prep the subscription config/filters to be sent to the server subscriptionConfig() diff --git a/cmd/syncPublishScreenAndServe.go b/cmd/syncPublishScreenAndServe.go index 6da80263..1288a3a4 100644 --- a/cmd/syncPublishScreenAndServe.go +++ b/cmd/syncPublishScreenAndServe.go @@ -49,6 +49,7 @@ func init() { func syncPublishScreenAndServe() { log.SetLevel(log.DebugLevel) + log.SetOutput(os.Stdout) blockChain, ethClient, rpcClient := getBlockChainAndClients() db := utils.LoadPostgres(databaseConfig, blockChain.Node()) @@ -93,7 +94,7 @@ func syncPublishScreenAndServe() { var wsEndpoint string wsEndpoint = viper.GetString("server.wsEndpoint") if wsEndpoint == "" { - wsEndpoint = "127.0.0.1:2019" + wsEndpoint = "127.0.0.1:80" } _, _, err = rpc.StartWSEndpoint(wsEndpoint, processor.APIs(), []string{"vulcanizedb"}, nil, true) if err != nil { diff --git a/dockerfiles/seed_node/Dockerfile b/dockerfiles/seed_node/Dockerfile index 3675fb19..fc809234 100644 --- a/dockerfiles/seed_node/Dockerfile +++ b/dockerfiles/seed_node/Dockerfile @@ -1,28 +1,37 @@ FROM golang:alpine as builder -RUN apk --update --no-cache add make git g++ +RUN apk --update --no-cache add dep make git g++ linux-headers # DEBUG RUN apk add busybox-extras -# Build statically linked vDB binary (wonky path because of Dep) +# Get and build vulcanizedb syncAndPublish fork +RUN go get -u -d github.com/vulcanize/vulcanizedb WORKDIR /go/src/github.com/vulcanize/vulcanizedb -ADD . . -RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' . +RUN git checkout syncAndPublish +RUN dep ensure +RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o vulcanizedb . -# Build statically linked IPFS binary (statically linked and wonky path because we need to use custom fork) +# Get and build vulcanize's go-ipfs fork +RUN go get -u -d github.com/ipfs/go-ipfs WORKDIR /go/src/github.com/ipfs/go-ipfs -ADD . . -RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' . +RUN git remote add vulcanize https://github.com/vulcanize/go-ipfs.git +RUN git fetch vulcanize +RUN git checkout -b pg_ipfs vulcanize/postgres_update +RUN dep ensure +RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o ipfs ./cmd/ipfs -# Build statically linked Geth binary (statically linked and wonky path because we need to use custom fork) +# Get and build vulcanize's geth fork +RUN go get -u -d github.com/ethereum/go-ethereum WORKDIR /go/src/github.com/ethereum/go-ethereum -ADD . . -RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' . +RUN git remote add vulcanize https://github.com/vulcanize/go-ethereum.git +RUN git fetch vulcanize +RUN git checkout -b statediff_geth vulcanize/rpc_statediffing +RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o geth ./cmd/geth # Build migration tool RUN go get -u -d github.com/pressly/goose/cmd/goose WORKDIR /go/src/github.com/pressly/goose/cmd/goose -RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -tags='no_mysql no_sqlite' -o goose +RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -tags='no_mysql no_sqlite' -o goose . WORKDIR /go/src/github.com/vulcanize/vulcanizedb @@ -31,16 +40,21 @@ FROM alpine WORKDIR /app ARG USER -ARG config_file=environments/example.toml -ARG vdb_command=syncPublishScreenAndServe -ARG vdb_pg_connect="postgres://$USER@/vulcanize_public?sslmode=disable" +ARG config_file=environments/syncPublishScreenAndServe.toml ARG vdb_dbname="vulcanize_public" +ARG vdb_hostname="localhost" +ARG vdb_port="5432" +ARG vdb_user="postgres" +ARG vdb_password # setup environment -ENV VDB_COMMAND="$vdb_command" -ENV VDB_PG_CONNECT="$vdb_pg_connect" +ENV VDB_PG_NAME="$vdb_dbname" +ENV VDB_PG_HOSTNAME="$vdb_hostname" +ENV VDB_PG_PORT="$vdb_port" +ENV VDB_PG_USER="$vdb_user" +ENV VDB_PG_PASSWORD="$vdb_password" -RUN adduser -Du 5000 $USER +RUN adduser -D 5000 $USER USER $USER # chown first so dir is writable @@ -52,10 +66,12 @@ COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/vulcanizedb/d COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/vulcanizedb vulcanizedb COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose goose COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/db/migrations migrations/vulcanizedb -COPY --from=builder /go/src/github.com/ipfs/go-ipfs ipfs -COPY --from=builder /go/src/github.com/ethereum/go-ethereum geth +COPY --from=builder /go/src/github.com/ipfs/go-ipfs/ipfs ipfs +COPY --from=builder /go/src/github.com/ethereum/go-ethereum/geth geth # DEBUG COPY --from=builder /usr/bin/telnet /bin/telnet +EXPOSE 80 + CMD ["./startup_script.sh"] diff --git a/dockerfiles/seed_node/startup_script.sh b/dockerfiles/seed_node/startup_script.sh index de8dac96..cacc1ce1 100755 --- a/dockerfiles/seed_node/startup_script.sh +++ b/dockerfiles/seed_node/startup_script.sh @@ -1,37 +1,71 @@ -#!/usr/bin/env bash -# Runs the migrations and starts the syncPublishScreenAndServe service +#!/bin/sh +# Runs the db migrations and starts the seed node services # Exit if the variable tests fail set -e # Check the database variables are set -test $DATABASE_NAME -test $DATABASE_HOSTNAME -test $DATABASE_PORT -test $DATABASE_USER -test $DATABASE_PASSWORD - -# Export our database variables so that the IPFS Postgres plugin can use them -export IPFS_PGHOST=$DATABASE_HOSTNAME -export IPFS_PGUSER=$DATABASE_USER -export IPFS_PGDATABASE=$DATABASE_NAME -export IPFS_PGPORT=$DATABASE_PORT -export IPFS_PGPASSWORD=$DATABASE_PASSWORD - -# Construct the connection string for postgres -CONNECT_STRING=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable -echo "Connecting with: $CONNECT_STRING" - +test $VDB_PG_NAME +test $VDB_PG_HOSTNAME +test $VDB_PG_PORT +test $VDB_PG_USER set +e +# Export our database variables so that the IPFS Postgres plugin can use them +export IPFS_PGHOST=$VDB_PG_HOSTNAME +export IPFS_PGUSER=$VDB_PG_USER +export IPFS_PGDATABASE=$VDB_PG_NAME +export IPFS_PGPORT=$VDB_PG_PORT +export IPFS_PGPASSWORD=$VDB_PG_PASSWORD + +# Construct the connection string for postgres +VDB_PG_CONNECT=postgresql://$VDB_PG_USER:$VDB_PG_PASSWORD@$VDB_PG_HOSTNAME:$VDB_PG_PORT/$VDB_PG_NAME?sslmode=disable + # Run the DB migrations -./goose postgres "$CONNECT_STRING" up +echo "Connecting with: $VDB_PG_CONNECT" +echo "Running database migrations" +./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up + +# If the db migrations ran without err if [ $? -eq 0 ]; then - # Fire up the services - ipfs ipfs init --profile=postgresds & - geth --statediff --statediff.streamblock --ws --syncmode=full & - ./vulcanizedb syncPublishScreenAndServe --config environments/seedNodeStaging.toml & + # Initialize PG-IPFS + echo "Initializing Postgres-IPFS profile" + ./ipfs init --profile=postgresds else - echo "Could not run migrations. Are the database details correct?" + echo "Could not run migrations. Are the database details correct?" + exit fi + +# If IPFS initialization was successful +if [ $? -eq 0 ]; then + # Begin the state-diffing Geth process + echo "Beginning the state-diffing Geth process" + ./geth --statediff --statediff.streamblock --ws --syncmode=full 2>&1 | tee -a log.txt & + sleep 1 +else + echo "Could not initialize Postgres backed IPFS profile. Are the database details correct?" + exit +fi + +# If Geth startup was successful +if [ $? -eq 0 ]; then + # Wait until block synchronisation has begun + echo "Waiting for block synchronization to begin" + ( tail -f -n0 log.txt & ) | grep -q "Block synchronisation started" # this blocks til we see "Block synchronisation started" + # And then spin up the syncPublishScreenAndServe Vulcanizedb service + echo "Beginning the syncPublishScreenAndServe vulcanizedb process" + ./vulcanizedb syncPublishScreenAndServe --config=config.toml 2>&1 | tee -a log.txt & +else + echo "Could not initialize Postgres backed IPFS profile. Are the database details correct?" + exit +fi + +# If Vulcanizedb startup was successful +if [ $? -eq 0 ]; then + echo "Seed node successfully booted" +else + echo "Could not start vulcanizedb syncPublishScreenAndServe process. Is the config file correct?" + exit +fi + wait diff --git a/environments/seedNodeSubscription.toml b/environments/seedNodeSubscription.toml new file mode 100644 index 00000000..ec208ebd --- /dev/null +++ b/environments/seedNodeSubscription.toml @@ -0,0 +1,40 @@ +[subscription] + path = "ws://127.0.0.1:80" + backfill = true + backfillOnly = false + startingBlock = 0 + endingBlock = 0 + [subscription.headerFilter] + off = false + finalOnly = true + [subscription.trxFilter] + off = false + src = [ + "0xde0B295669a9FD93d5F28D9Ec85E40f4cb697BAe", + ] + dst = [ + "0xde0B295669a9FD93d5F28D9Ec85E40f4cb697BAe", + ] + [subscription.receiptFilter] + off = false + topic0s = [ + "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", + "0x930a61a57a70a73c2a503615b87e2e54fe5b9cdeacda518270b852296ab1a377" + ] + [subscription.stateFilter] + off = false + addresses = [ + "0xde0B295669a9FD93d5F28D9Ec85E40f4cb697BAe" + ] + intermediateNodes = false + [subscription.storageFilter] + off = true + addresses = [ + "", + "" + ] + storageKeys = [ + "", + "" + ] + intermediateNodes = false \ No newline at end of file diff --git a/environments/syncPublishScreenAndServe.toml b/environments/syncPublishScreenAndServe.toml index 5d234e40..002bae4b 100644 --- a/environments/syncPublishScreenAndServe.toml +++ b/environments/syncPublishScreenAndServe.toml @@ -1,5 +1,5 @@ [database] - name = "vulcanize_demo" + name = "vulcanize_public" hostname = "localhost" port = 5432 @@ -9,45 +9,4 @@ [server] ipcPath = "/Users/iannorden/.vulcanize/vulcanize.ipc" - wsEndpoint = "127.0.0.1:2019" - -[subscription] - path = "ws://127.0.0.1:2019" - backfill = true - backfillOnly = false - startingBlock = 0 - endingBlock = 0 - [subscription.headerFilter] - off = false - finalOnly = true - [subscription.trxFilter] - off = false - src = [ - "0xde0B295669a9FD93d5F28D9Ec85E40f4cb697BAe", - ] - dst = [ - "0xde0B295669a9FD93d5F28D9Ec85E40f4cb697BAe", - ] - [subscription.receiptFilter] - off = false - topic0s = [ - "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", - "0x930a61a57a70a73c2a503615b87e2e54fe5b9cdeacda518270b852296ab1a377" - ] - [subscription.stateFilter] - off = false - addresses = [ - "0xde0B295669a9FD93d5F28D9Ec85E40f4cb697BAe" - ] - intermediateNodes = false - [subscription.storageFilter] - off = true - addresses = [ - "", - "" - ] - storageKeys = [ - "", - "" - ] - intermediateNodes = false \ No newline at end of file + wsEndpoint = "127.0.0.1:80" diff --git a/vendor/github.com/ipfs/go-ipfs/plugin/loader/loader.go b/vendor/github.com/ipfs/go-ipfs/plugin/loader/loader.go index 6194b2c0..b5a2d08e 100644 --- a/vendor/github.com/ipfs/go-ipfs/plugin/loader/loader.go +++ b/vendor/github.com/ipfs/go-ipfs/plugin/loader/loader.go @@ -30,7 +30,6 @@ type PluginLoader struct { func NewPluginLoader(pluginDir string) (*PluginLoader, error) { plMap := make(map[string]plugin.Plugin) for _, v := range preloadPlugins { - println(v.Name()) plMap[v.Name()] = v }