diff --git a/.gitignore b/.gitignore index 15004028..1d7fa1f9 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,7 @@ Vagrantfile vagrant*.sh .vagrant test_scripts/ -ipfs-blockchain-watcher +ipld-eth-server postgraphile/build/ postgraphile/node_modules/ postgraphile/package-lock.json diff --git a/README.md b/README.md index 306ed2a1..607308b2 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ -# ipfs-blockchain-watcher +# ipld-eth-server -[![Go Report Card](https://goreportcard.com/badge/github.com/vulcanize/ipfs-blockchain-watcher)](https://goreportcard.com/report/github.com/vulcanize/ipfs-blockchain-watcher) +[![Go Report Card](https://goreportcard.com/badge/github.com/vulcanize/ipld-eth-server)](https://goreportcard.com/report/github.com/vulcanize/ipld-eth-server) -> ipfs-blockchain-watcher is used to extract, transform, and load all eth or btc data into an IPFS-backing Postgres datastore while generating useful secondary indexes around the data in other Postgres tables +> ipld-eth-server is used to extract, transform, and load all eth or btc data into an IPFS-backing Postgres datastore while generating useful secondary indexes around the data in other Postgres tables ## Table of Contents 1. [Background](#background) @@ -13,13 +13,13 @@ 1. [License](#license) ## Background -ipfs-blockchain-watcher is a collection of interfaces that are used to extract, process, store, and index -all blockchain data in Postgres-IPFS. The raw data indexed by ipfs-blockchain-watcher serves as the basis for more specific watchers and applications. +ipld-eth-server is a collection of interfaces that are used to extract, process, store, and index +all blockchain data in Postgres-IPFS. The raw data indexed by ipld-eth-server serves as the basis for more specific watchers and applications. Currently the service supports complete processing of all Bitcoin and Ethereum data. ## Architecture -More details on the design of ipfs-blockchain-watcher can be found in [here](./documentation/architecture.md) +More details on the design of ipld-eth-server can be found in [here](./documentation/architecture.md) ## Dependencies Minimal build dependencies @@ -49,7 +49,7 @@ is required for running the automated tests and is used by the `make migrate` co 1. [Install Postgres](https://wiki.postgresql.org/wiki/Detailed_installation_guides) 1. Create a superuser for yourself and make sure `psql --list` works without prompting for a password. 1. `createdb vulcanize_public` -1. `cd $GOPATH/src/github.com/vulcanize/ipfs-blockchain-watcher` +1. `cd $GOPATH/src/github.com/vulcanize/ipld-eth-server` 1. Run the migrations: `make migrate HOST_NAME=localhost NAME=vulcanize_public PORT=5432` - There are optional vars `USER=username:password` if the database user is not the default user `postgres` and/or a password is present - To rollback a single step: `make rollback NAME=vulcanize_public` @@ -68,7 +68,7 @@ Data is stored in an [IPFS-backing Postgres datastore](https://github.com/ipfs/g By default data is written directly to the ipfs blockstore in Postgres; the public.blocks table. In this case no further IPFS configuration is needed at this time. -Optionally, ipfs-blockchain-watcher can be configured to function through an internal ipfs node interface using the flag: `-ipfs-mode=interface`. +Optionally, ipld-eth-server can be configured to function through an internal ipfs node interface using the flag: `-ipfs-mode=interface`. Operating through the ipfs interface provides the option to configure a block exchange that can search remotely for IPLD data found missing in the local datastore. This option is irrelevant in most cases and this mode has some disadvantages, namely: @@ -79,7 +79,7 @@ This option is irrelevant in most cases and this mode has some disadvantages, na More information for configuring Postgres-IPFS can be found [here](./documentation/ipfs.md) ### Blockchain -This section describes how to setup an Ethereum or Bitcoin node to serve as a data source for ipfs-blockchain-watcher +This section describes how to setup an Ethereum or Bitcoin node to serve as a data source for ipld-eth-server #### Ethereum For Ethereum, [a special fork of go-ethereum](https://github.com/vulcanize/go-ethereum/tree/statediff_at_anyblock-1.9.11) is currently *requirde*. @@ -122,7 +122,7 @@ The default ws url is "127.0.0.1:8546" and the default http url is "127.0.0.1:85 These values will be used as the `ethereum.wsPath` and `ethereum.httpPath` in the config, respectively. #### Bitcoin -For Bitcoin, ipfs-blockchain-watcher is able to operate entirely through the universally exposed JSON-RPC interfaces. +For Bitcoin, ipld-eth-server is able to operate entirely through the universally exposed JSON-RPC interfaces. This means any of the standard full nodes can be used (e.g. bitcoind, btcd) as the data source. Point at a remote node or set one up locally using the instructions for [bitcoind](https://github.com/bitcoin/bitcoin) and [btcd](https://github.com/btcsuite/btcd). @@ -133,11 +133,11 @@ The default http url is "127.0.0.1:8332". We will use the http endpoint as both ### Watcher Finally, setup the watcher process itself. -Start by downloading ipfs-blockchain-watcher and moving into the repo: +Start by downloading ipld-eth-server and moving into the repo: -`GO111MODULE=off go get -d github.com/vulcanize/ipfs-blockchain-watcher` +`GO111MODULE=off go get -d github.com/vulcanize/ipld-eth-server` -`cd $GOPATH/src/github.com/vulcanize/ipfs-blockchain-watcher` +`cd $GOPATH/src/github.com/vulcanize/ipld-eth-server` Then, build the binary: @@ -146,11 +146,11 @@ Then, build the binary: ## Usage After building the binary, run as -`./ipfs-blockchain-watcher watch --config=` +`./ipld-eth-server watch --config=` ### Configuration -Below is the set of universal config parameters for the ipfs-blockchain-watcher command, in .toml form, with the respective environmental variables commented to the side. +Below is the set of universal config parameters for the ipld-eth-server command, in .toml form, with the respective environmental variables commented to the side. This set of parameters needs to be set no matter the chain type. ```toml @@ -207,7 +207,7 @@ For Ethereum: ``` ### Exposing the data -A number of different APIs for remote access to ipfs-blockchain-watcher data can be exposed, these are discussed in more detail [here](./documentation/apis.md) +A number of different APIs for remote access to ipld-eth-server data can be exposed, these are discussed in more detail [here](./documentation/apis.md) ### Testing `make test` will run the unit tests diff --git a/dockerfiles/migrations/Dockerfile b/dockerfiles/migrations/Dockerfile index dbcea6d0..bce16c9d 100644 --- a/dockerfiles/migrations/Dockerfile +++ b/dockerfiles/migrations/Dockerfile @@ -7,15 +7,15 @@ RUN apk add busybox-extras # this is probably a noob move, but I want apk from alpine for the above but need to avoid Go 1.13 below as this error still occurs https://github.com/ipfs/go-ipfs/issues/6603 FROM golang:1.12.4 as builder -# Get and build ipfs-blockchain-watcher -ADD . /go/src/github.com/vulcanize/ipfs-blockchain-watcher +# Get and build ipld-eth-server +ADD . /go/src/github.com/vulcanize/ipld-eth-server # Build migration tool RUN go get -u -d github.com/pressly/goose/cmd/goose WORKDIR /go/src/github.com/pressly/goose/cmd/goose RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -tags='no_mysql no_sqlite' -o goose . -WORKDIR /go/src/github.com/vulcanize/ipfs-blockchain-watcher +WORKDIR /go/src/github.com/vulcanize/ipld-eth-server # app container FROM alpine @@ -29,12 +29,12 @@ USER $USER # chown first so dir is writable # note: using $USER is merged, but not in the stable release yet -COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/ipfs-blockchain-watcher/dockerfiles/migrations/startup_script.sh . +COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/ipld-eth-server/dockerfiles/migrations/startup_script.sh . # keep binaries immutable COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose goose -COPY --from=builder /go/src/github.com/vulcanize/ipfs-blockchain-watcher/db/migrations migrations/vulcanizedb +COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-server/db/migrations migrations/vulcanizedb # XXX dir is already writeable RUN touch vulcanizedb.log CMD ["./startup_script.sh"] \ No newline at end of file diff --git a/dockerfiles/super_node/Dockerfile b/dockerfiles/super_node/Dockerfile index 03c7aded..df9c6f1e 100644 --- a/dockerfiles/super_node/Dockerfile +++ b/dockerfiles/super_node/Dockerfile @@ -4,10 +4,10 @@ RUN apk --update --no-cache add make git g++ linux-headers # DEBUG RUN apk add busybox-extras -# Get and build ipfs-blockchain-watcher -ADD . /go/src/github.com/vulcanize/ipfs-blockchain-watcher -WORKDIR /go/src/github.com/vulcanize/ipfs-blockchain-watcher -RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o ipfs-blockchain-watcher . +# Get and build ipld-eth-server +ADD . /go/src/github.com/vulcanize/ipld-eth-server +WORKDIR /go/src/github.com/vulcanize/ipld-eth-server +RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o ipld-eth-server . # Build migration tool WORKDIR / @@ -15,7 +15,7 @@ RUN go get -u -d github.com/pressly/goose/cmd/goose WORKDIR /go/src/github.com/pressly/goose/cmd/goose RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -tags='no_mysql no_sqlite' -o goose . -WORKDIR /go/src/github.com/vulcanize/ipfs-blockchain-watcher +WORKDIR /go/src/github.com/vulcanize/ipld-eth-server # app container FROM alpine @@ -32,16 +32,16 @@ USER $USER # chown first so dir is writable # note: using $USER is merged, but not in the stable release yet -COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/ipfs-blockchain-watcher/$CONFIG_FILE config.toml -COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/ipfs-blockchain-watcher/dockerfiles/super_node/startup_script.sh . -COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/ipfs-blockchain-watcher/dockerfiles/super_node/entrypoint.sh . +COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/ipld-eth-server/$CONFIG_FILE config.toml +COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/ipld-eth-server/dockerfiles/super_node/startup_script.sh . +COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/ipld-eth-server/dockerfiles/super_node/entrypoint.sh . # keep binaries immutable -COPY --from=builder /go/src/github.com/vulcanize/ipfs-blockchain-watcher/ipfs-blockchain-watcher ipfs-blockchain-watcher +COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-server/ipld-eth-server ipld-eth-server COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose goose -COPY --from=builder /go/src/github.com/vulcanize/ipfs-blockchain-watcher/db/migrations migrations/vulcanizedb -COPY --from=builder /go/src/github.com/vulcanize/ipfs-blockchain-watcher/environments environments +COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-server/db/migrations migrations/vulcanizedb +COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-server/environments environments EXPOSE $EXPOSE_PORT_1 EXPOSE $EXPOSE_PORT_2 diff --git a/dockerfiles/super_node/entrypoint.sh b/dockerfiles/super_node/entrypoint.sh index ff93bc60..b203311a 100755 --- a/dockerfiles/super_node/entrypoint.sh +++ b/dockerfiles/super_node/entrypoint.sh @@ -35,7 +35,7 @@ echo "Beginning the vulcanizedb process" VDB_CONFIG_FILE=${VDB_CONFIG_FILE:-config.toml} DEFAULT_OPTIONS="--config=$VDB_CONFIG_FILE" VDB_FULL_CL=${VDB_FULL_CL:-$VDB_COMMAND $DEFAULT_OPTIONS} -echo running: ./ipfs-blockchain-watcher $VDB_FULL_CL $@ +echo running: ./ipld-eth-server $VDB_FULL_CL $@ case "$1" in "/bin/sh" ) @@ -49,8 +49,8 @@ if [[ -z "$vdb_args" ]]; then vdb_args="--config=config.toml" fi -echo running: ./ipfs-blockchain-watcher $vdb_args -./ipfs-blockchain-watcher $vdb_args +echo running: ./ipld-eth-server $vdb_args +./ipld-eth-server $vdb_args rv=$? if [ $rv != 0 ]; then diff --git a/dockerfiles/super_node/startup_script.sh b/dockerfiles/super_node/startup_script.sh index 3ff9ca02..326b1baf 100755 --- a/dockerfiles/super_node/startup_script.sh +++ b/dockerfiles/super_node/startup_script.sh @@ -49,7 +49,7 @@ fi # If IPFS initialization was successful if [[ $? -eq 0 ]]; then echo "Running the VulcanizeDB process" - ./ipfs-blockchain-watcher ${VDB_COMMAND} --config=config.toml + ./ipld-eth-server ${VDB_COMMAND} --config=config.toml else echo "Could not initialize IPFS." exit 1 diff --git a/scripts/reset_db b/scripts/reset_db index f6fefe3b..0a57f8d3 100755 --- a/scripts/reset_db +++ b/scripts/reset_db @@ -11,9 +11,9 @@ fi db=$1 dir=$(basename "$(pwd)") -if [ $dir != "ipfs-blockchain-watcher" ] +if [ $dir != "ipld-eth-server" ] then - echo "Run me from the ipfs-blockchain-watcher root dir" + echo "Run me from the ipld-eth-server root dir" exit 1 fi diff --git a/test_config/test_config.go b/test_config/test_config.go index 96143b04..7236422e 100644 --- a/test_config/test_config.go +++ b/test_config/test_config.go @@ -34,7 +34,7 @@ func init() { func setTestConfig() { vip := viper.New() vip.SetConfigName("testing") - vip.AddConfigPath("$GOPATH/src/github.com/vulcanize/ipfs-blockchain-watcher/environments/") + vip.AddConfigPath("$GOPATH/src/github.com/vulcanize/ipld-eth-server/environments/") if err := vip.ReadInConfig(); err != nil { logrus.Fatal(err) } diff --git a/utils/utils.go b/utils/utils.go index 73e70708..56771295 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -22,9 +22,9 @@ import ( "github.com/sirupsen/logrus" "github.com/vulcanize/ipfs-blockchain-watcher/pkg/config" - "github.com/vulcanize/ipfs-blockchain-watcher/pkg/node" + "github.com/vulcanize/ipld-eth-server/pkg/node" "github.com/vulcanize/ipfs-blockchain-watcher/pkg/postgres" - "github.com/vulcanize/ipfs-blockchain-watcher/pkg/shared" + "github.com/vulcanize/ipld-eth-server/pkg/shared" ) func LoadPostgres(database config.Database, node node.Node) postgres.DB { diff --git a/utils/utils_test.go b/utils/utils_test.go index b6c865d0..b36ac66c 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -20,7 +20,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/vulcanize/ipfs-blockchain-watcher/utils" + "github.com/vulcanize/ipld-eth-server/utils" ) var _ = Describe("GetBlockHeightBins", func() {