diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index a97d1f64..00000000 --- a/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -FROM golang:alpine as builder -RUN apk --update --no-cache add make git g++ - -# Build statically linked vDB binary (wonky path because of Dep) -RUN mkdir -p /go/src/github.com/vulcanize/ipfs-blockchain-watcher -ADD . /go/src/github.com/vulcanize/ipfs-blockchain-watcher -WORKDIR /go/src/github.com/vulcanize/ipfs-blockchain-watcher -RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' . - -# Build migration tool -RUN go get -u -d github.com/pressly/goose/cmd/goose -WORKDIR /go/src/github.com/pressly/goose/cmd/goose -RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -tags='no_mysql no_sqlite' -o goose - -# Second stage -FROM alpine -COPY --from=builder /go/src/github.com/vulcanize/ipfs-blockchain-watcher/vulcanizedb /app/vulcanizedb -COPY --from=builder /go/src/github.com/vulcanize/ipfs-blockchain-watcher/environments/staging.toml /app/environments/ -COPY --from=builder /go/src/github.com/vulcanize/ipfs-blockchain-watcher/dockerfiles/startup_script.sh /app/ -COPY --from=builder /go/src/github.com/vulcanize/ipfs-blockchain-watcher/db/migrations/* /app/ -COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose /app/goose - -WORKDIR /app -CMD ["./startup_script.sh"] diff --git a/Makefile b/Makefile index fc031472..9848db9b 100644 --- a/Makefile +++ b/Makefile @@ -105,7 +105,7 @@ rollback: $(GOOSE) checkdbvars pg_dump -O -s $(CONNECT_STRING) > db/schema.sql -## Rollbackt to a select migration (id/timestamp) +## Rollback to a select migration (id/timestamp) .PHONY: rollback_to rollback_to: $(GOOSE) checkmigration checkdbvars $(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" down-to "$(MIGRATION)" @@ -137,26 +137,3 @@ version_migrations: import: test -n "$(NAME)" # $$NAME psql $(NAME) < db/schema.sql - - -# Docker actions -## Rinkeby docker environment -RINKEBY_COMPOSE_FILE=dockerfiles/rinkeby/docker-compose.yml - -.PHONY: rinkeby_env_up -rinkeby_env_up: - docker-compose -f $(RINKEBY_COMPOSE_FILE) up -d geth - docker-compose -f $(RINKEBY_COMPOSE_FILE) up --build migrations - docker-compose -f $(RINKEBY_COMPOSE_FILE) up -d --build vulcanizedb - -.PHONY: rinkeby_env_deploy -rinkeby_env_deploy: - docker-compose -f $(RINKEBY_COMPOSE_FILE) up -d --build vulcanizedb - -.PHONY: dev_env_migrate -rinkeby_env_migrate: - docker-compose -f $(RINKEBY_COMPOSE_FILE) up --build migrations - -.PHONY: rinkeby_env_down -rinkeby_env_down: - docker-compose -f $(RINKEBY_COMPOSE_FILE) down diff --git a/README.md b/README.md index e0eca85e..620aaf59 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![Go Report Card](https://goreportcard.com/badge/github.com/vulcanize/ipfs-blockchain-watcher)](https://goreportcard.com/report/github.com/vulcanize/ipfs-blockchain-watcher) -> Tool for extracting and indexing blockchain data on PG-IPFS +> ipfs-blockchain-watcher is used to extract, transform, and load all eth or btc data into an IPFS-backing Postgres datastore while generating useful secondary indexes around that data in other Postgres tables ## Table of Contents 1. [Background](#background) @@ -30,7 +30,7 @@ More details on the design of ipfs-blockchain-watcher can be found in [here](./d ### Goose [goose](https://github.com/pressly/goose) is used for migration management. While it is not necessary to use `goose` for manual setup, it -is required for running the automated tests and by the `make migrate` command. +is required for running the automated tests and is used by the `make migrate` command. ### Postgres 1. [Install Postgres](https://wiki.postgresql.org/wiki/Detailed_installation_guides) @@ -52,12 +52,12 @@ localhost. To allow access on Ubuntu, set localhost connections via hostname, ip ### IPFS Data is stored in an [IPFS-backing Postgres datastore](https://github.com/ipfs/go-ds-sql). -By default data is writen directly to the ipfs blockstore in Postgres; the public.blocks table. +By default data is written directly to the ipfs blockstore in Postgres; the public.blocks table. In this case no further IPFS configuration is needed at this time. Optionally, ipfs-blockchain-watcher can be configured to function through an internal ipfs node interface using the flag: `-ipfs-mode=interface`. Operating through the ipfs interface provides the option to configure a block exchange that can search remotely for IPLD data found missing in the local datastore. -This option is irrelevant in most cases and this mode some disadvantages, namely: +This option is irrelevant in most cases and this mode has some disadvantages, namely: 1. Environment must have IPFS configured 1. Process will contend with the lockfile at `$IPFS_PATH` @@ -120,7 +120,7 @@ The default http url is "127.0.0.1:8332". We will use the http endpoint as both ### Watcher Finally, setup the watcher process itself. -Start by downloading vulcanizedb and moving into the repo: +Start by downloading ipfs-blockchain-watcher and moving into the repo: `go get github.com/vulcanize/ipfs-blockchain-watcher` diff --git a/dockerfiles/README.md b/dockerfiles/README.md deleted file mode 100644 index b2fbf2bb..00000000 --- a/dockerfiles/README.md +++ /dev/null @@ -1,32 +0,0 @@ -S -`Dockerfile` will build an alpine image containing: -- vDB as a binary with runtime deps statically linked: `/app/vulcanizedb` -- The migration tool goose: `/app/goose` -- Two services for running `headerSync` and `continuousLogSync`, started with the default configuration `environments/staging.toml`. - -By default, vDB is configured towards the Kovan deploy. The configuration values can be overridden using environment variables, using the same hierarchical naming pattern but in CAPS and using underscores. For example, the contract address for the `Pit` can be set with the variable `CONTRACT_ADDRESS_PIT="0x123..."`. - -## To use the container: -1. Setup a postgres database with superuser `vulcanize` -2. Set the env variables `DATABASE_NAME`, `DATABASE_HOSTNAME`, - `DATABASE_PORT`, `DATABASE_USER` & `DATABASE_PASSWORD` -3. Run the DB migrations: - * `./goose postgres "postgresql://$(DATABASE_USER):$(DATABASE_PASSWORD)@$(DATABASE_HOSTNAME):$(DATABASE_PORT)/$(DATABASE_NAME)?sslmode=disable" -e` -4. Set `CLIENT_IPCPATH` to a node endpoint -5. Set the contract variables: - * `CONTRACT_ADDRESS_[CONTRACT NAME]=0x123...` - * `CONTRACT_ABI_[CONTRACT NAME]="ABI STRING"` - * `CONTRACT_DEPLOYMENT-BLOCK_[CONTRACT NAME]=0` (doesn't really matter on a short chain, just avoids long unnecessary searching) -6. Start the `headerSync` and `continuousLogSync` services: - * `./vulcanizedb headerSync --config environments/staging.toml` - * `./vulcanizedb continuousLogSync --config environments/staging.toml` - -### Automated -The steps above have been rolled into a script: `/app/startup_script.sh`, which just assumes the DB env variables have been set, and defaults the rest to Kovan according to `environments/staging.toml`. This can be called with something like: - -`docker run -d -e DATABASE_NAME=vulcanize_public -e DATABASE_HOSTNAME=localhost -e DATABASE_PORT=5432 -e DATABASE_USER=vulcanize -e DATABASE_PASSWORD=vulcanize m0ar/images:vDB` - -### Logging -When running, vDB services log to `/app/vulcanizedb.log`. - diff --git a/dockerfiles/migrations/Dockerfile b/dockerfiles/migrations/Dockerfile index 220f5580..dbcea6d0 100644 --- a/dockerfiles/migrations/Dockerfile +++ b/dockerfiles/migrations/Dockerfile @@ -7,7 +7,7 @@ RUN apk add busybox-extras # this is probably a noob move, but I want apk from alpine for the above but need to avoid Go 1.13 below as this error still occurs https://github.com/ipfs/go-ipfs/issues/6603 FROM golang:1.12.4 as builder -# Get and build vulcanizedb +# Get and build ipfs-blockchain-watcher ADD . /go/src/github.com/vulcanize/ipfs-blockchain-watcher # Build migration tool diff --git a/dockerfiles/startup_script.sh b/dockerfiles/startup_script.sh deleted file mode 100755 index de74e711..00000000 --- a/dockerfiles/startup_script.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh -# Runs the migrations and starts the headerSync and continuousLogSync services - -# Exit if the variable tests fail -set -e - -# Check the database variables are set -test $DATABASE_NAME -test $DATABASE_HOSTNAME -test $DATABASE_PORT -test $DATABASE_USER -test $DATABASE_PASSWORD - -# Construct the connection string for postgres -CONNECT_STRING=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable -echo "Connecting with: $CONNECT_STRING" - -set +e - -# Run the DB migrations -./goose postgres "$CONNECT_STRING" up -if [ $? -eq 0 ]; then - # Fire up the services - ./vulcanizedb headerSync --config environments/staging.toml & - ./vulcanizedb continuousLogSync --config environments/staging.toml & -else - echo "Could not run migrations. Are the database details correct?" -fi -wait diff --git a/dockerfiles/super_node/Dockerfile b/dockerfiles/super_node/Dockerfile index 69679db1..ef6a3cf7 100644 --- a/dockerfiles/super_node/Dockerfile +++ b/dockerfiles/super_node/Dockerfile @@ -4,10 +4,10 @@ RUN apk --update --no-cache add make git g++ linux-headers # DEBUG RUN apk add busybox-extras -# Get and build vulcanizedb +# Get and build ipfs-blockchain-watcher ADD . /go/src/github.com/vulcanize/ipfs-blockchain-watcher WORKDIR /go/src/github.com/vulcanize/ipfs-blockchain-watcher -RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o vulcanizedb . +RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o ipfs-blockchain-watcher . # Build migration tool RUN go get -u -d github.com/pressly/goose/cmd/goose @@ -37,7 +37,7 @@ COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/ipfs-blockcha # keep binaries immutable -COPY --from=builder /go/src/github.com/vulcanize/ipfs-blockchain-watcher/vulcanizedb vulcanizedb +COPY --from=builder /go/src/github.com/vulcanize/ipfs-blockchain-watcher/ipfs-blockchain-watcher ipfs-blockchain-watcher COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose goose COPY --from=builder /go/src/github.com/vulcanize/ipfs-blockchain-watcher/db/migrations migrations/vulcanizedb COPY --from=builder /go/src/github.com/vulcanize/ipfs-blockchain-watcher/environments environments diff --git a/dockerfiles/super_node/entrypoint.sh b/dockerfiles/super_node/entrypoint.sh index 1af5a9cb..7df52bc5 100755 --- a/dockerfiles/super_node/entrypoint.sh +++ b/dockerfiles/super_node/entrypoint.sh @@ -36,7 +36,7 @@ echo "Beginning the vulcanizedb process" VDB_CONFIG_FILE=${VDB_CONFIG_FILE:-config.toml} DEFAULT_OPTIONS="--config=$VDB_CONFIG_FILE" VDB_FULL_CL=${VDB_FULL_CL:-$VDB_COMMAND $DEFAULT_OPTIONS} -echo running: ./vulcanizedb $VDB_FULL_CL $@ +echo running: ./ipfs-blockchain-watcher $VDB_FULL_CL $@ case "$1" in "/bin/sh" ) @@ -50,8 +50,8 @@ if [[ -z "$vdb_args" ]]; then vdb_args="--config=config.toml" fi -echo running: ./vulcanizedb $vdb_args -./vulcanizedb $vdb_args +echo running: ./ipfs-blockchain-watcher $vdb_args +./ipfs-blockchain-watcher $vdb_args rv=$? if [ $rv != 0 ]; then diff --git a/dockerfiles/super_node/startup_script.sh b/dockerfiles/super_node/startup_script.sh index e30c5069..89039372 100755 --- a/dockerfiles/super_node/startup_script.sh +++ b/dockerfiles/super_node/startup_script.sh @@ -50,7 +50,7 @@ fi # If IPFS initialization was successful if [[ $? -eq 0 ]]; then echo "Running the VulcanizeDB process" - ./vulcanizedb ${VDB_COMMAND} --config=config.toml + ./ipfs-blockchain-watcher ${VDB_COMMAND} --config=config.toml else echo "Could not initialize IPFS." exit 1 diff --git a/scripts/reset_db b/scripts/reset_db index c7870271..f6fefe3b 100755 --- a/scripts/reset_db +++ b/scripts/reset_db @@ -11,9 +11,9 @@ fi db=$1 dir=$(basename "$(pwd)") -if [ $dir != "vulcanizedb" ] +if [ $dir != "ipfs-blockchain-watcher" ] then - echo "Run me from the vulcanizedb root dir" + echo "Run me from the ipfs-blockchain-watcher root dir" exit 1 fi