diff --git a/cmd/resync.go b/cmd/resync.go
index af16a58e..a2f6cf65 100644
--- a/cmd/resync.go
+++ b/cmd/resync.go
@@ -19,6 +19,7 @@ import (
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/viper"
+ "github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"github.com/vulcanize/vulcanizedb/pkg/super_node/resync"
@@ -45,8 +46,10 @@ func rsyncCmdCommand() {
logWithCommand.Fatal(err)
}
logWithCommand.Infof("resync config: %+v", rConfig)
- if err := ipfs.InitIPFSPlugins(); err != nil {
- logWithCommand.Fatal(err)
+ if rConfig.IPFSMode == shared.LocalInterface {
+ if err := ipfs.InitIPFSPlugins(); err != nil {
+ logWithCommand.Fatal(err)
+ }
}
logWithCommand.Debug("initializing new resync service")
rService, err := resync.NewResyncService(rConfig)
diff --git a/cmd/superNode.go b/cmd/superNode.go
index e2cfdd08..02f35420 100644
--- a/cmd/superNode.go
+++ b/cmd/superNode.go
@@ -18,11 +18,10 @@ package cmd
import (
"sync"
- "github.com/spf13/viper"
-
"github.com/ethereum/go-ethereum/rpc"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
+ "github.com/spf13/viper"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"github.com/vulcanize/vulcanizedb/pkg/super_node"
@@ -60,8 +59,10 @@ func superNode() {
logWithCommand.Fatal(err)
}
logWithCommand.Infof("super node config: %+v", superNodeConfig)
- if err := ipfs.InitIPFSPlugins(); err != nil {
- logWithCommand.Fatal(err)
+ if superNodeConfig.IPFSMode == shared.LocalInterface {
+ if err := ipfs.InitIPFSPlugins(); err != nil {
+ logWithCommand.Fatal(err)
+ }
}
wg := &sync.WaitGroup{}
logWithCommand.Debug("initializing new super node service")
diff --git a/cmd/watch.go b/cmd/watch.go
deleted file mode 100644
index 0978e425..00000000
--- a/cmd/watch.go
+++ /dev/null
@@ -1,43 +0,0 @@
-// Copyright © 2020 Vulcanize, Inc
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see .
-
-package cmd
-
-import (
- "fmt"
-
- "github.com/spf13/cobra"
-)
-
-// watchCmd represents the watch command
-var watchCmd = &cobra.Command{
- Use: "watch",
- Short: "Watch and transform data from a chain source",
- Long: `This command allows one to configure a set of wasm functions and SQL trigger functions
-that call them to watch and transform data from the specified chain source.
-
-A watcher is composed of four parts:
-1) Go execution engine- this command- which fetches raw chain data and adds it to the Postres queued ready data tables
-2) TOML config file which specifies what subset of chain data to fetch and from where and contains references to the below
-3) Set of WASM binaries which are loaded into Postgres and used by
-4) Set of PostgreSQL trigger functions which automatically act on data as it is inserted into the queued ready data tables`,
- Run: func(cmd *cobra.Command, args []string) {
- fmt.Println("watch called")
- },
-}
-
-func init() {
- rootCmd.AddCommand(watchCmd)
-}
diff --git a/dockerfiles/super_node/Dockerfile b/dockerfiles/super_node/Dockerfile
index 84ebe54e..2c29c455 100644
--- a/dockerfiles/super_node/Dockerfile
+++ b/dockerfiles/super_node/Dockerfile
@@ -1,25 +1,14 @@
-FROM golang:alpine
+FROM golang:1.12-alpine as builder
RUN apk --update --no-cache add make git g++ linux-headers
# DEBUG
RUN apk add busybox-extras
-# this is probably a noob move, but I want apk from alpine for the above but need to avoid Go 1.13 below as this error still occurs https://github.com/ipfs/go-ipfs/issues/6603
-FROM golang:1.12.4 as builder
-
# Get and build vulcanizedb
ADD . /go/src/github.com/vulcanize/vulcanizedb
WORKDIR /go/src/github.com/vulcanize/vulcanizedb
RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o vulcanizedb .
-# Get and build vulcanize's go-ipfs fork
-RUN go get -u -d github.com/ipfs/go-ipfs
-WORKDIR /go/src/github.com/ipfs/go-ipfs
-RUN git remote add vulcanize https://github.com/vulcanize/go-ipfs.git
-RUN git fetch vulcanize
-RUN git checkout -b pg_ipfs vulcanize/postgres_update
-RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o ipfs ./cmd/ipfs
-
# Build migration tool
RUN go get -u -d github.com/pressly/goose/cmd/goose
WORKDIR /go/src/github.com/pressly/goose/cmd/goose
@@ -52,7 +41,6 @@ COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/vulcanizedb vulcani
COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose goose
COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/db/migrations migrations/vulcanizedb
COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/environments environments
-COPY --from=builder /go/src/github.com/ipfs/go-ipfs/ipfs ipfs
EXPOSE $EXPOSE_PORT_1
EXPOSE $EXPOSE_PORT_2
diff --git a/dockerfiles/super_node/docker-compose.yml b/dockerfiles/super_node/docker-compose.yml
index b12a514e..4487be47 100644
--- a/dockerfiles/super_node/docker-compose.yml
+++ b/dockerfiles/super_node/docker-compose.yml
@@ -29,8 +29,6 @@ services:
CONFIG_FILE: ./environments/superNodeBTC.toml
environment:
VDB_COMMAND: "superNode"
- IPFS_INIT: "true"
- IPFS_PATH: "/root/.btc/.ipfs"
DATABASE_NAME: "vulcanize_public"
DATABASE_HOSTNAME: "db"
DATABASE_PORT: 5432
@@ -54,8 +52,6 @@ services:
CONFIG_FILE: ./environments/superNodeETH.toml
environment:
VDB_COMMAND: "superNode"
- IPFS_INIT: "true"
- IPFS_PATH: "/root/.eth/.ipfs"
DATABASE_NAME: "vulcanize_public"
DATABASE_HOSTNAME: "db"
DATABASE_PORT: 5432
diff --git a/dockerfiles/super_node/entrypoint.sh b/dockerfiles/super_node/entrypoint.sh
index 77fa4778..a5f9ae7b 100755
--- a/dockerfiles/super_node/entrypoint.sh
+++ b/dockerfiles/super_node/entrypoint.sh
@@ -31,26 +31,6 @@ if [ $rv != 0 ]; then
exit 1
fi
-# Export our database variables so that the IPFS Postgres plugin can use them
-export IPFS_PGHOST=$DATABASE_HOSTNAME
-export IPFS_PGUSER=$DATABASE_USER
-export IPFS_PGDATABASE=$DATABASE_NAME
-export IPFS_PGPORT=$DATABASE_PORT
-export IPFS_PGPASSWORD=$DATABASE_PASSWORD
-
-
-if [ ! -d "$HOME/.ipfs" ]; then
- # initialize PG-IPFS
- echo "Initializing Postgres-IPFS profile"
- ./ipfs init --profile=postgresds
-
- rv=$?
- if [ $rv != 0 ]; then
- echo "Could not initialize ipfs"
- exit 1
- fi
-fi
-
echo "Beginning the vulcanizedb process"
VDB_CONFIG_FILE=${VDB_CONFIG_FILE:-config.toml}
@@ -58,12 +38,19 @@ DEFAULT_OPTIONS="--config=$VDB_CONFIG_FILE"
VDB_FULL_CL=${VDB_FULL_CL:-$VDB_COMMAND $DEFAULT_OPTIONS}
echo running: ./vulcanizedb $VDB_FULL_CL $@
+case "$1" in
+ "/bin/sh" )
+ echo dropping to shell
+ exec /bin/sh
+esac
+
vdb_args="$@"
# default is to use the config passed by the build arg
-if [[ -z "$vdb_args" ]];
+if [[ -z "$vdb_args" ]]; then
vdb_args="--config=config.toml"
fi
+echo running: ./vulcanizedb $vdb_args
./vulcanizedb $vdb_args
rv=$?
diff --git a/go.mod b/go.mod
index 17360377..116cca35 100644
--- a/go.mod
+++ b/go.mod
@@ -32,6 +32,7 @@ require (
github.com/ipfs/go-ipfs-blockstore v0.0.1
github.com/ipfs/go-ipfs-cmds v0.1.1 // indirect
github.com/ipfs/go-ipfs-config v0.0.3 // indirect
+ github.com/ipfs/go-ipfs-ds-help v0.0.1
github.com/ipfs/go-ipfs-exchange-interface v0.0.1
github.com/ipfs/go-ipld-cbor v0.0.3 // indirect
github.com/ipfs/go-ipld-format v0.0.2
diff --git a/go.sum b/go.sum
index dae0ac07..ea4249a2 100644
--- a/go.sum
+++ b/go.sum
@@ -17,7 +17,6 @@ github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxB
github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM=
github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
-github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
@@ -31,7 +30,6 @@ github.com/VictoriaMetrics/fastcache v1.5.3/go.mod h1:+jv9Ckb+za/P1ZRg/sulP5Ni1v
github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8=
github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ=
github.com/aristanetworks/goarista v0.0.0-20190712234253-ed1100a1c015 h1:7ABPr1+uJdqESAdlVevnc/2FJGiC/K3uMg1JiELeF+0=
@@ -62,7 +60,6 @@ github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtE
github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
-github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk=
github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s=
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
@@ -129,16 +126,12 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI=
github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI=
github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
-github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
-github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
-github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk=
github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
@@ -148,7 +141,6 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
@@ -162,15 +154,12 @@ github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c h1:7lF+Vz0LqiRidnzC1Oq86fpX1q/iEv2KJdrCtttYjT4=
github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989 h1:giknQ4mEuDFmmHSrGcbargOuLHQGtywqo4mheITex54=
github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
@@ -199,7 +188,6 @@ github.com/huin/goupnp v0.0.0-20180415215157-1395d1447324/go.mod h1:MZ2ZmwcBpvOo
github.com/huin/goupnp v1.0.0 h1:wg75sLpL6DZqwHQN6E1Cfk6mtfzS45z8OV+ic+DtHRo=
github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc=
github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o=
-github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY=
github.com/ipfs/bbloom v0.0.1 h1:s7KkiBPfxCeDVo47KySjK0ACPc5GJRUxFpdyWEuDjhw=
@@ -230,7 +218,6 @@ github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAK
github.com/ipfs/go-datastore v0.0.3/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE=
github.com/ipfs/go-datastore v0.0.5 h1:q3OfiOZV5rlsK1H5V8benjeUApRfMGs4Mrhmr6NriQo=
github.com/ipfs/go-datastore v0.0.5/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE=
-github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk=
github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps=
github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjvRcKeSGij8=
github.com/ipfs/go-ds-badger v0.0.5 h1:dxKuqw5T1Jm8OuV+lchA76H9QZFyPKZeLuT6bN42hJQ=
@@ -246,11 +233,9 @@ github.com/ipfs/go-fs-lock v0.0.1 h1:XHX8uW4jQBYWHj59XXcjg7BHlHxV9ZOYs6Y43yb7/l0
github.com/ipfs/go-fs-lock v0.0.1/go.mod h1:DNBekbboPKcxs1aukPSaOtFA3QfSdi5C855v0i9XJ8Y=
github.com/ipfs/go-ipfs-blockstore v0.0.1 h1:O9n3PbmTYZoNhkgkEyrXTznbmktIXif62xLX+8dPHzc=
github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08=
-github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ=
github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk=
github.com/ipfs/go-ipfs-chunker v0.0.1 h1:cHUUxKFQ99pozdahi+uSC/3Y6HeRpi9oTeUHbE27SEw=
github.com/ipfs/go-ipfs-chunker v0.0.1/go.mod h1:tWewYK0we3+rMbOh7pPFGDyypCtvGcBFymgY4rSDLAw=
-github.com/ipfs/go-ipfs-cmds v0.1.1 h1:H9/BLf5rcsULHMj/x8gC0e5o+raYhqk1OQsfzbGMNM4=
github.com/ipfs/go-ipfs-cmds v0.1.1/go.mod h1:k1zMXcOLtljA9iAnZHddbH69yVm5+weRL0snmMD/rK0=
github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=
github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ=
@@ -262,7 +247,6 @@ github.com/ipfs/go-ipfs-exchange-interface v0.0.1/go.mod h1:c8MwfHjtQjPoDyiy9cFq
github.com/ipfs/go-ipfs-exchange-offline v0.0.1 h1:P56jYKZF7lDDOLx5SotVh5KFxoY6C81I1NSHW1FxGew=
github.com/ipfs/go-ipfs-exchange-offline v0.0.1/go.mod h1:WhHSFCVYX36H/anEKQboAzpUws3x7UeEGkzQc3iNkM0=
github.com/ipfs/go-ipfs-files v0.0.2/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4=
-github.com/ipfs/go-ipfs-files v0.0.3 h1:ME+QnC3uOyla1ciRPezDW0ynQYK2ikOh9OCKAEg4uUA=
github.com/ipfs/go-ipfs-files v0.0.3/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4=
github.com/ipfs/go-ipfs-files v0.0.4 h1:WzRCivcybUQch/Qh6v8LBRhKtRsjnwyiuOV09mK7mrE=
github.com/ipfs/go-ipfs-files v0.0.4/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4=
@@ -316,12 +300,10 @@ github.com/ipfs/interface-go-ipfs-core v0.1.0/go.mod h1:h1zJvvfh9dcNU0bK+Jag516L
github.com/jackpal/gateway v1.0.4/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA=
github.com/jackpal/gateway v1.0.5 h1:qzXWUJfuMdlLMtt0a3Dgt+xkWQiA5itDEITVJtuSwMc=
github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA=
-github.com/jackpal/go-nat-pmp v1.0.1 h1:i0LektDkO1QlrTm/cSuP+PyBCDnYvjPLGl4LdWEMiaA=
github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 h1:6OvNmYgJyexcZ3pYbTI9jWx5tHo1Dee/tWbLMfPe2TA=
github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
github.com/jbenet/go-cienv v0.0.0-20150120210510-1bb1476777ec/go.mod h1:rGaEvXB4uRSZMmzKNLoXvTu1sfx+1kv/DojUlPrSZGs=
-github.com/jbenet/go-cienv v0.1.0 h1:Vc/s0QbQtoxX8MwwSLWWh+xNNZvM3Lw7NsTcHrvvhMc=
github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA=
github.com/jbenet/go-is-domain v1.0.2 h1:11r5MSptcNFZyBoqubBQnVMUKRWLuRjL1banaIk+iYo=
github.com/jbenet/go-is-domain v1.0.2/go.mod h1:xbRLRb0S7FgzDBTJlguhDVwLYM/5yNtvktxj2Ttfy7Q=
@@ -338,7 +320,6 @@ github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22
github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
-github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/julienschmidt/httprouter v1.1.1-0.20170430222011-975b5c4c7c21/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
@@ -348,16 +329,12 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b h1:wxtKgYHEncAU00muMD06dzLiahtGM1eouRNOzVV7tdQ=
github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk=
-github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
-github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
@@ -392,7 +369,6 @@ github.com/libp2p/go-libp2p-autonat-svc v0.1.0 h1:28IM7iWMDclZeVkpiFQaWVANwXwE7z
github.com/libp2p/go-libp2p-autonat-svc v0.1.0/go.mod h1:fqi8Obl/z3R4PFVLm8xFtZ6PBL9MlV/xumymRFkKq5A=
github.com/libp2p/go-libp2p-blankhost v0.0.1/go.mod h1:Ibpbw/7cPPYwFb7PACIWdvxxv0t0XCCI10t7czjAjTc=
github.com/libp2p/go-libp2p-blankhost v0.1.1/go.mod h1:pf2fvdLJPsC1FsVrNP3DUUvMzUts2dsLLBEpo1vW1ro=
-github.com/libp2p/go-libp2p-blankhost v0.1.3 h1:0KycuXvPDhmehw0ASsg+s1o3IfXgCUDqfzAl94KEBOg=
github.com/libp2p/go-libp2p-blankhost v0.1.3/go.mod h1:KML1//wiKR8vuuJO0y3LUd1uLv+tlkGTAr3jC0S5cLg=
github.com/libp2p/go-libp2p-circuit v0.0.1/go.mod h1:Dqm0s/BiV63j8EEAs8hr1H5HudqvCAeXxDyic59lCwE=
github.com/libp2p/go-libp2p-circuit v0.0.9/go.mod h1:uU+IBvEQzCu953/ps7bYzC/D/R0Ho2A9LfKVVCatlqU=
@@ -439,7 +415,6 @@ github.com/libp2p/go-libp2p-nat v0.0.4/go.mod h1:N9Js/zVtAXqaeT99cXgTV9e75KpnWCv
github.com/libp2p/go-libp2p-net v0.0.1/go.mod h1:Yt3zgmlsHOgUWSXmt5V/Jpz9upuJBE8EgNU9DrCcR8c=
github.com/libp2p/go-libp2p-net v0.0.2/go.mod h1:Yt3zgmlsHOgUWSXmt5V/Jpz9upuJBE8EgNU9DrCcR8c=
github.com/libp2p/go-libp2p-netutil v0.0.1/go.mod h1:GdusFvujWZI9Vt0X5BKqwWWmZFxecf9Gt03cKxm2f/Q=
-github.com/libp2p/go-libp2p-netutil v0.1.0 h1:zscYDNVEcGxyUpMd0JReUZTrpMfia8PmLKcKF72EAMQ=
github.com/libp2p/go-libp2p-netutil v0.1.0/go.mod h1:3Qv/aDqtMLTUyQeundkKsA+YCThNdbQD54k3TqjpbFU=
github.com/libp2p/go-libp2p-peer v0.0.1/go.mod h1:nXQvOBbwVqoP+T5Y5nCjeH4sP9IX/J0AMzcDUVruVoo=
github.com/libp2p/go-libp2p-peer v0.1.1/go.mod h1:jkF12jGB4Gk/IOo+yomm+7oLWxF278F7UnrYUQ1Q8es=
@@ -482,7 +457,6 @@ github.com/libp2p/go-libp2p-swarm v0.1.1/go.mod h1:4NVJaLwq/dr5kEq79Jo6pMin7ZFwL
github.com/libp2p/go-libp2p-testing v0.0.1/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E=
github.com/libp2p/go-libp2p-testing v0.0.2/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E=
github.com/libp2p/go-libp2p-testing v0.0.3/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E=
-github.com/libp2p/go-libp2p-testing v0.0.4 h1:Qev57UR47GcLPXWjrunv5aLIQGO4n9mhI/8/EIrEEFc=
github.com/libp2p/go-libp2p-testing v0.0.4/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E=
github.com/libp2p/go-libp2p-tls v0.1.0 h1:o4bjjAdnUjNgJoPoDd0wUaZH7K+EenlNWJpgyXB3ulA=
github.com/libp2p/go-libp2p-tls v0.1.0/go.mod h1:VZdoSWQDeNpIIAFJFv+6uqTqpnIIDHcqZQSTC/A1TT0=
@@ -559,7 +533,6 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd
github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y=
github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
-github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/4=
github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/miekg/dns v1.1.4/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
@@ -610,7 +583,6 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0=
github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
-github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88=
github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c h1:1RHs3tNxjXGHeul8z2t6H2N2TlAqpKe5yryJztRx4Jk=
github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
@@ -634,7 +606,6 @@ github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssy
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o=
github.com/polydawn/refmt v0.0.0-20190408063855-01bf1e26dd14/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o=
@@ -660,7 +631,6 @@ github.com/prometheus/tsdb v0.10.0/go.mod h1:oi49uRhEe9dPUTlS3JRZOwJuVi6tmh10QSg
github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho=
github.com/rjeczalik/notify v0.9.2 h1:MiTWrPj55mNDHEiIX5YUSKefw/+lCQVoAFmD6oQm5w8=
github.com/rjeczalik/notify v0.9.2/go.mod h1:aErll2f0sUX9PXZnVNyeiObbmTlk5jnMoCa4QEjJeqM=
-github.com/robertkrimen/otto v0.0.0-20170205013659-6a77b7cbc37d/go.mod h1:xvqspoSXJTIpemEonrMDFq6XzwHYYgToXWj5eRX1OtY=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
@@ -673,15 +643,11 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
-github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8=
github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU=
-github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs=
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
-github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a h1:/eS3yfGjQKG+9kayBkj0ip1BGhq6zJ3eaVksphxAaek=
github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a/go.mod h1:7AyxJNCJ7SBZ1MfVQCWD6Uqo2oubI2Eq2y2eqf+A5r0=
-github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU=
github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spaolacci/murmur3 v1.0.1-0.20190317074736-539464a789e9/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
@@ -713,9 +679,7 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d h1:gZZadD8H+fF+n9CmNhYL1Y0dJB+kLOmKd7FbPJLeGHs=
github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA=
@@ -726,8 +690,6 @@ github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:s
github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
-github.com/vulcanize/go-ethereum v1.5.10-0.20200116224441-2a980ec3dcb8 h1:BHt0OW0rTgndFjSju7brF3dPceXWQuEV0IdtY8BjjT8=
-github.com/vulcanize/go-ethereum v1.5.10-0.20200116224441-2a980ec3dcb8/go.mod h1:a9TqabFudpDu1nucId+k9S8R9whYaHnGBLKFouA5EAo=
github.com/vulcanize/go-ethereum v1.5.10-0.20200311182536-d07dc803d290 h1:uMWt+x6JhVT7GyL983weZSxv1zDBxvGlI9HNkcTnUeg=
github.com/vulcanize/go-ethereum v1.5.10-0.20200311182536-d07dc803d290/go.mod h1:7oC0Ni6dosMv5pxMigm6s0hN8g4haJMBnqmmo0D9YfQ=
github.com/vulcanize/go-ipfs v0.4.22-alpha h1:W+6njT14KWllMhABRFtPndqHw8SHCt5SqD4YX528kxM=
@@ -735,7 +697,6 @@ github.com/vulcanize/go-ipfs v0.4.22-alpha/go.mod h1:uaekWWeoaA0A9Dv1LObOKCSh9kI
github.com/vulcanize/go-ipfs-config v0.0.8-alpha h1:peaFvbEcPShF6ymOd8flqKkFz4YfcrNr/UOO7FmbWoQ=
github.com/vulcanize/go-ipfs-config v0.0.8-alpha/go.mod h1:IGkVTacurWv9WFKc7IBPjHGM/7hi6+PEClqUb/l2BIM=
github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
-github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830 h1:8kxMKmKzXXL4Ru1nyhvdms/JjWt+3YLpvRb/bAjO/y0=
github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc h1:BCPnHtcboadS0DvysUuJXZ4lWVv5Bh5i7+tbIyi+ck4=
github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc/go.mod h1:r45hJU7yEoA81k6MWNhpMj/kms0n14dkzkxYHoB96UM=
@@ -776,7 +737,6 @@ go.uber.org/dig v1.7.0 h1:E5/L92iQTNJTjfgJF2KgU+/JpMaiuvK2DHLBj0+kSZk=
go.uber.org/dig v1.7.0/go.mod h1:z+dSd2TP9Usi48jL8M3v63iSBVkiwtVyMKxMZYYauPg=
go.uber.org/fx v1.9.0 h1:7OAz8ucp35AU8eydejpYG7QrbE8rLKzGhHbZlJi5LYY=
go.uber.org/fx v1.9.0/go.mod h1:mFdUyAUuJ3w4jAckiKSKbldsxy1ojpAMJ+dVZg5Y0Aw=
-go.uber.org/goleak v1.0.0 h1:qsup4IcBdlmsnGfqyLl4Ntn3C2XCCuKAE7DwHpScyUo=
go.uber.org/goleak v1.0.0/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
@@ -802,7 +762,6 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs=
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/net v0.0.0-20180524181706-dfa909b99c79/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -863,13 +822,11 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20191108193012-7d206e10da11 h1:Yq9t9jnGoR+dBuitxdo9l6Q7xh/zOyNnYUtDKaQ3x0E=
golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
-google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
@@ -880,13 +837,10 @@ google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiq
google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
-gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU=
gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c=
-gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190213234257-ec84240a7772 h1:hhsSf/5z74Ck/DJYc+R8zpq8KGm7uJvpdLRQED/IedA=
gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190213234257-ec84240a7772/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns=
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78=
diff --git a/libraries/shared/storage/backfiller.go b/libraries/shared/storage/backfiller.go
index 6584e38d..08e3fdea 100644
--- a/libraries/shared/storage/backfiller.go
+++ b/libraries/shared/storage/backfiller.go
@@ -27,6 +27,7 @@ import (
"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+ "github.com/vulcanize/vulcanizedb/libraries/shared/utilities"
)
const (
@@ -63,7 +64,7 @@ func (bf *backFiller) BackFill(startingBlock, endingBlock uint64, backFill chan
logrus.Infof("going to fill in gap from %d to %d", startingBlock, endingBlock)
// break the range up into bins of smaller ranges
- blockRangeBins, err := utils.GetBlockHeightBins(startingBlock, endingBlock, bf.batchSize)
+ blockRangeBins, err := utilities.GetBlockHeightBins(startingBlock, endingBlock, bf.batchSize)
if err != nil {
return err
}
diff --git a/libraries/shared/storage/utils/bins.go b/libraries/shared/storage/utils/bins.go
deleted file mode 100644
index 42ea0a84..00000000
--- a/libraries/shared/storage/utils/bins.go
+++ /dev/null
@@ -1,72 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see .
-
-package utils
-
-import (
- "errors"
-
- "github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
-)
-
-func GetBlockHeightBins(startingBlock, endingBlock, batchSize uint64) ([][]uint64, error) {
- if endingBlock < startingBlock {
- return nil, errors.New("backfill: ending block number needs to be greater than starting block number")
- }
- if batchSize == 0 {
- return nil, errors.New("backfill: batchsize needs to be greater than zero")
- }
- length := endingBlock - startingBlock + 1
- numberOfBins := length / batchSize
- remainder := length % batchSize
- if remainder != 0 {
- numberOfBins++
- }
- blockRangeBins := make([][]uint64, numberOfBins)
- for i := range blockRangeBins {
- nextBinStart := startingBlock + batchSize
- blockRange := make([]uint64, 0, nextBinStart-startingBlock+1)
- for j := startingBlock; j < nextBinStart && j <= endingBlock; j++ {
- blockRange = append(blockRange, j)
- }
- startingBlock = nextBinStart
- blockRangeBins[i] = blockRange
- }
- return blockRangeBins, nil
-}
-
-func MissingHeightsToGaps(heights []uint64) []shared.Gap {
- validationGaps := make([]shared.Gap, 0)
- start := heights[0]
- lastHeight := start
- for i, height := range heights[1:] {
- if height != lastHeight+1 {
- validationGaps = append(validationGaps, shared.Gap{
- Start: start,
- Stop: lastHeight,
- })
- start = height
- }
- if i+2 == len(heights) {
- validationGaps = append(validationGaps, shared.Gap{
- Start: start,
- Stop: height,
- })
- }
- lastHeight = height
- }
- return validationGaps
-}
diff --git a/libraries/shared/utilities/utilities_suite_test.go b/libraries/shared/utilities/utilities_suite_test.go
new file mode 100644
index 00000000..cfcc2707
--- /dev/null
+++ b/libraries/shared/utilities/utilities_suite_test.go
@@ -0,0 +1,36 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package utilities_test
+
+import (
+ "io/ioutil"
+ "testing"
+
+ "github.com/sirupsen/logrus"
+
+ . "github.com/onsi/ginkgo"
+ . "github.com/onsi/gomega"
+)
+
+func TestShared(t *testing.T) {
+ RegisterFailHandler(Fail)
+ RunSpecs(t, "Shared Utilities Suite")
+}
+
+var _ = BeforeSuite(func() {
+ logrus.SetOutput(ioutil.Discard)
+})
diff --git a/libraries/shared/utilities/utils.go b/libraries/shared/utilities/utils.go
index 02fc104e..3f586213 100644
--- a/libraries/shared/utilities/utils.go
+++ b/libraries/shared/utilities/utils.go
@@ -16,9 +16,62 @@
package utilities
-func NullToZero(str string) string {
- if str == "" {
- return "0"
+import (
+ "errors"
+
+ "github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+)
+
+// GetBlockHeightBins splits a block range up into bins of block heights of the given batch size
+func GetBlockHeightBins(startingBlock, endingBlock, batchSize uint64) ([][]uint64, error) {
+ if endingBlock < startingBlock {
+ return nil, errors.New("backfill: ending block number needs to be greater than starting block number")
}
- return str
+ if batchSize == 0 {
+ return nil, errors.New("backfill: batchsize needs to be greater than zero")
+ }
+ length := endingBlock - startingBlock + 1
+ numberOfBins := length / batchSize
+ remainder := length % batchSize
+ if remainder != 0 {
+ numberOfBins++
+ }
+ blockRangeBins := make([][]uint64, numberOfBins)
+ for i := range blockRangeBins {
+ nextBinStart := startingBlock + batchSize
+ blockRange := make([]uint64, 0, nextBinStart-startingBlock+1)
+ for j := startingBlock; j < nextBinStart && j <= endingBlock; j++ {
+ blockRange = append(blockRange, j)
+ }
+ startingBlock = nextBinStart
+ blockRangeBins[i] = blockRange
+ }
+ return blockRangeBins, nil
+}
+
+// MissingHeightsToGaps returns a slice of gaps from a slice of missing block heights
+func MissingHeightsToGaps(heights []uint64) []shared.Gap {
+ if len(heights) == 0 {
+ return nil
+ }
+ validationGaps := make([]shared.Gap, 0)
+ start := heights[0]
+ lastHeight := start
+ for i, height := range heights[1:] {
+ if height != lastHeight+1 {
+ validationGaps = append(validationGaps, shared.Gap{
+ Start: start,
+ Stop: lastHeight,
+ })
+ start = height
+ }
+ if i+2 == len(heights) {
+ validationGaps = append(validationGaps, shared.Gap{
+ Start: start,
+ Stop: height,
+ })
+ }
+ lastHeight = height
+ }
+ return validationGaps
}
diff --git a/libraries/shared/storage/utils/bins_test.go b/libraries/shared/utilities/utils_test.go
similarity index 96%
rename from libraries/shared/storage/utils/bins_test.go
rename to libraries/shared/utilities/utils_test.go
index 19985b3b..8a3050ef 100644
--- a/libraries/shared/storage/utils/bins_test.go
+++ b/libraries/shared/utilities/utils_test.go
@@ -14,13 +14,13 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see .
-package utils_test
+package utilities_test
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
- "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+ utils "github.com/vulcanize/vulcanizedb/libraries/shared/utilities"
)
var _ = Describe("GetBlockHeightBins", func() {
diff --git a/pkg/super_node/backfiller.go b/pkg/super_node/backfiller.go
index 2684df69..48a89110 100644
--- a/pkg/super_node/backfiller.go
+++ b/pkg/super_node/backfiller.go
@@ -24,7 +24,7 @@ import (
log "github.com/sirupsen/logrus"
- "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+ utils "github.com/vulcanize/vulcanizedb/libraries/shared/utilities"
"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
)
@@ -69,11 +69,11 @@ type BackFillService struct {
// NewBackFillService returns a new BackFillInterface
func NewBackFillService(settings *Config, screenAndServeChan chan shared.ConvertedData) (BackFillInterface, error) {
- publisher, err := NewIPLDPublisher(settings.Chain, settings.IPFSPath)
+ publisher, err := NewIPLDPublisher(settings.Chain, settings.IPFSPath, settings.DB, settings.IPFSMode)
if err != nil {
return nil, err
}
- indexer, err := NewCIDIndexer(settings.Chain, settings.DB)
+ indexer, err := NewCIDIndexer(settings.Chain, settings.DB, settings.IPFSMode)
if err != nil {
return nil, err
}
diff --git a/pkg/super_node/btc/retriever.go b/pkg/super_node/btc/cid_retriever.go
similarity index 91%
rename from pkg/super_node/btc/retriever.go
rename to pkg/super_node/btc/cid_retriever.go
index 350917bb..ce6f7630 100644
--- a/pkg/super_node/btc/retriever.go
+++ b/pkg/super_node/btc/cid_retriever.go
@@ -17,17 +17,16 @@
package btc
import (
+ "database/sql"
"fmt"
"math/big"
- "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
-
- "github.com/lib/pq"
-
"github.com/ethereum/go-ethereum/common"
"github.com/jmoiron/sqlx"
+ "github.com/lib/pq"
log "github.com/sirupsen/logrus"
+ utils "github.com/vulcanize/vulcanizedb/libraries/shared/utilities"
"github.com/vulcanize/vulcanizedb/pkg/postgres"
"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
)
@@ -65,17 +64,26 @@ func (ecr *CIDRetriever) Retrieve(filter shared.SubscriptionSettings, blockNumbe
return nil, true, fmt.Errorf("btc retriever expected filter type %T got %T", &SubscriptionSettings{}, filter)
}
log.Debug("retrieving cids")
+
+ // Begin new db tx
tx, err := ecr.db.Beginx()
if err != nil {
return nil, true, err
}
+ defer func() {
+ if p := recover(); p != nil {
+ shared.Rollback(tx)
+ panic(p)
+ } else if err != nil {
+ shared.Rollback(tx)
+ } else {
+ err = tx.Commit()
+ }
+ }()
// Retrieve cached header CIDs
headers, err := ecr.RetrieveHeaderCIDs(tx, blockNumber)
if err != nil {
- if err := tx.Rollback(); err != nil {
- log.Error(err)
- }
log.Error("header cid retrieval error")
return nil, true, err
}
@@ -92,9 +100,6 @@ func (ecr *CIDRetriever) Retrieve(filter shared.SubscriptionSettings, blockNumbe
if !streamFilter.TxFilter.Off {
cw.Transactions, err = ecr.RetrieveTxCIDs(tx, streamFilter.TxFilter, header.ID)
if err != nil {
- if err := tx.Rollback(); err != nil {
- log.Error(err)
- }
log.Error("transaction cid retrieval error")
return nil, true, err
}
@@ -105,7 +110,7 @@ func (ecr *CIDRetriever) Retrieve(filter shared.SubscriptionSettings, blockNumbe
cws[i] = cw
}
- return cws, empty, tx.Commit()
+ return cws, empty, err
}
// RetrieveHeaderCIDs retrieves and returns all of the header cids at the provided blockheight
@@ -173,7 +178,7 @@ func (ecr *CIDRetriever) RetrieveGapsInData(validationLevel int) ([]shared.Gap,
Start uint64 `db:"start"`
Stop uint64 `db:"stop"`
}, 0)
- if err := ecr.db.Select(&results, pgStr); err != nil {
+ if err := ecr.db.Select(&results, pgStr); err != nil && err != sql.ErrNoRows {
return nil, err
}
emptyGaps := make([]shared.Gap, len(results))
@@ -190,53 +195,66 @@ func (ecr *CIDRetriever) RetrieveGapsInData(validationLevel int) ([]shared.Gap,
WHERE times_validated < $1
ORDER BY block_number`
var heights []uint64
- if err := ecr.db.Select(&heights, pgStr, validationLevel); err != nil {
+ if err := ecr.db.Select(&heights, pgStr, validationLevel); err != nil && err != sql.ErrNoRows {
return nil, err
}
- if len(heights) == 0 {
- return emptyGaps, nil
- }
return append(emptyGaps, utils.MissingHeightsToGaps(heights)...), nil
}
// RetrieveBlockByHash returns all of the CIDs needed to compose an entire block, for a given block hash
func (ecr *CIDRetriever) RetrieveBlockByHash(blockHash common.Hash) (HeaderModel, []TxModel, error) {
log.Debug("retrieving block cids for block hash ", blockHash.String())
+
+ // Begin new db tx
tx, err := ecr.db.Beginx()
if err != nil {
return HeaderModel{}, nil, err
}
+ defer func() {
+ if p := recover(); p != nil {
+ shared.Rollback(tx)
+ panic(p)
+ } else if err != nil {
+ shared.Rollback(tx)
+ } else {
+ err = tx.Commit()
+ }
+ }()
+
headerCID, err := ecr.RetrieveHeaderCIDByHash(tx, blockHash)
if err != nil {
- if err := tx.Rollback(); err != nil {
- log.Error(err)
- }
log.Error("header cid retrieval error")
return HeaderModel{}, nil, err
}
txCIDs, err := ecr.RetrieveTxCIDsByHeaderID(tx, headerCID.ID)
if err != nil {
- if err := tx.Rollback(); err != nil {
- log.Error(err)
- }
log.Error("tx cid retrieval error")
- return HeaderModel{}, nil, err
}
- return headerCID, txCIDs, tx.Commit()
+ return headerCID, txCIDs, err
}
// RetrieveBlockByNumber returns all of the CIDs needed to compose an entire block, for a given block number
func (ecr *CIDRetriever) RetrieveBlockByNumber(blockNumber int64) (HeaderModel, []TxModel, error) {
log.Debug("retrieving block cids for block number ", blockNumber)
+
+ // Begin new db tx
tx, err := ecr.db.Beginx()
if err != nil {
return HeaderModel{}, nil, err
}
+ defer func() {
+ if p := recover(); p != nil {
+ shared.Rollback(tx)
+ panic(p)
+ } else if err != nil {
+ shared.Rollback(tx)
+ } else {
+ err = tx.Commit()
+ }
+ }()
+
headerCID, err := ecr.RetrieveHeaderCIDs(tx, blockNumber)
if err != nil {
- if err := tx.Rollback(); err != nil {
- log.Error(err)
- }
log.Error("header cid retrieval error")
return HeaderModel{}, nil, err
}
@@ -245,13 +263,9 @@ func (ecr *CIDRetriever) RetrieveBlockByNumber(blockNumber int64) (HeaderModel,
}
txCIDs, err := ecr.RetrieveTxCIDsByHeaderID(tx, headerCID[0].ID)
if err != nil {
- if err := tx.Rollback(); err != nil {
- log.Error(err)
- }
log.Error("tx cid retrieval error")
- return HeaderModel{}, nil, err
}
- return headerCID[0], txCIDs, tx.Commit()
+ return headerCID[0], txCIDs, err
}
// RetrieveHeaderCIDByHash returns the header for the given block hash
diff --git a/pkg/super_node/btc/cleaner.go b/pkg/super_node/btc/cleaner.go
index a057dddf..98591765 100644
--- a/pkg/super_node/btc/cleaner.go
+++ b/pkg/super_node/btc/cleaner.go
@@ -50,9 +50,7 @@ func (c *Cleaner) ResetValidation(rngs [][2]uint64) error {
SET times_validated = 0
WHERE block_number BETWEEN $1 AND $2`
if _, err := tx.Exec(pgStr, rng[0], rng[1]); err != nil {
- if err := tx.Rollback(); err != nil {
- logrus.Error(err)
- }
+ shared.Rollback(tx)
return err
}
}
@@ -68,9 +66,7 @@ func (c *Cleaner) Clean(rngs [][2]uint64, t shared.DataType) error {
for _, rng := range rngs {
logrus.Infof("btc db cleaner cleaning up block range %d to %d", rng[0], rng[1])
if err := c.clean(tx, rng, t); err != nil {
- if err := tx.Rollback(); err != nil {
- logrus.Error(err)
- }
+ shared.Rollback(tx)
return err
}
}
diff --git a/pkg/super_node/btc/indexer.go b/pkg/super_node/btc/indexer.go
index a6dda9cf..ed869320 100644
--- a/pkg/super_node/btc/indexer.go
+++ b/pkg/super_node/btc/indexer.go
@@ -43,29 +43,42 @@ func (in *CIDIndexer) Index(cids shared.CIDsForIndexing) error {
if !ok {
return fmt.Errorf("btc indexer expected cids type %T got %T", &CIDPayload{}, cids)
}
+
+ // Begin new db tx
tx, err := in.db.Beginx()
if err != nil {
return err
}
- headerID, err := in.indexHeaderCID(tx, cidWrapper.HeaderCID, in.db.NodeID)
+ defer func() {
+ if p := recover(); p != nil {
+ shared.Rollback(tx)
+ panic(p)
+ } else if err != nil {
+ shared.Rollback(tx)
+ } else {
+ err = tx.Commit()
+ }
+ }()
+
+ headerID, err := in.indexHeaderCID(tx, cidWrapper.HeaderCID)
if err != nil {
logrus.Error("btc indexer error when indexing header")
return err
}
- if err := in.indexTransactionCIDs(tx, cidWrapper.TransactionCIDs, headerID); err != nil {
+ err = in.indexTransactionCIDs(tx, cidWrapper.TransactionCIDs, headerID)
+ if err != nil {
logrus.Error("btc indexer error when indexing transactions")
- return err
}
- return tx.Commit()
+ return err
}
-func (in *CIDIndexer) indexHeaderCID(tx *sqlx.Tx, header HeaderModel, nodeID int64) (int64, error) {
+func (in *CIDIndexer) indexHeaderCID(tx *sqlx.Tx, header HeaderModel) (int64, error) {
var headerID int64
err := tx.QueryRowx(`INSERT INTO btc.header_cids (block_number, block_hash, parent_hash, cid, timestamp, bits, node_id, times_validated)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
ON CONFLICT (block_number, block_hash) DO UPDATE SET (parent_hash, cid, timestamp, bits, node_id, times_validated) = ($3, $4, $5, $6, $7, btc.header_cids.times_validated + 1)
RETURNING id`,
- header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, header.Timestamp, header.Bits, nodeID, 1).Scan(&headerID)
+ header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, header.Timestamp, header.Bits, in.db.NodeID, 1).Scan(&headerID)
return headerID, err
}
diff --git a/pkg/super_node/btc/ipld_fetcher.go b/pkg/super_node/btc/ipld_fetcher.go
index c839e031..06dccc71 100644
--- a/pkg/super_node/btc/ipld_fetcher.go
+++ b/pkg/super_node/btc/ipld_fetcher.go
@@ -40,6 +40,7 @@ type IPLDFetcher struct {
}
// NewIPLDFetcher creates a pointer to a new IPLDFetcher
+// It interfaces with PG-IPFS through an internalized IPFS node interface
func NewIPLDFetcher(ipfsPath string) (*IPLDFetcher, error) {
blockService, err := ipfs.InitIPFSBlockService(ipfsPath)
if err != nil {
diff --git a/pkg/super_node/btc/ipld_pg_fetcher.go b/pkg/super_node/btc/ipld_pg_fetcher.go
new file mode 100644
index 00000000..53ea63e3
--- /dev/null
+++ b/pkg/super_node/btc/ipld_pg_fetcher.go
@@ -0,0 +1,107 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package btc
+
+import (
+ "fmt"
+
+ "github.com/jmoiron/sqlx"
+ log "github.com/sirupsen/logrus"
+
+ "github.com/vulcanize/vulcanizedb/pkg/ipfs"
+ "github.com/vulcanize/vulcanizedb/pkg/postgres"
+ "github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+)
+
+// IPLDPGFetcher satisfies the IPLDFetcher interface for ethereum
+// it interfaces directly with PG-IPFS instead of going through a node-interface or remote node
+type IPLDPGFetcher struct {
+ db *postgres.DB
+}
+
+// NewIPLDPGFetcher creates a pointer to a new IPLDPGFetcher
+func NewIPLDPGFetcher(db *postgres.DB) *IPLDPGFetcher {
+ return &IPLDPGFetcher{
+ db: db,
+ }
+}
+
+// Fetch is the exported method for fetching and returning all the IPLDS specified in the CIDWrapper
+func (f *IPLDPGFetcher) Fetch(cids shared.CIDsForFetching) (shared.IPLDs, error) {
+ cidWrapper, ok := cids.(*CIDWrapper)
+ if !ok {
+ return nil, fmt.Errorf("btc fetcher: expected cids type %T got %T", &CIDWrapper{}, cids)
+ }
+ log.Debug("fetching iplds")
+ iplds := IPLDs{}
+ iplds.BlockNumber = cidWrapper.BlockNumber
+
+ tx, err := f.db.Beginx()
+ if err != nil {
+ return nil, err
+ }
+ defer func() {
+ if p := recover(); p != nil {
+ shared.Rollback(tx)
+ panic(p)
+ } else if err != nil {
+ shared.Rollback(tx)
+ } else {
+ err = tx.Commit()
+ }
+ }()
+
+ iplds.Header, err = f.FetchHeader(tx, cidWrapper.Header)
+ if err != nil {
+ return nil, fmt.Errorf("btc pg fetcher: header fetching error: %s", err.Error())
+ }
+ iplds.Transactions, err = f.FetchTrxs(tx, cidWrapper.Transactions)
+ if err != nil {
+ return nil, fmt.Errorf("btc pg fetcher: transaction fetching error: %s", err.Error())
+ }
+ return iplds, err
+}
+
+// FetchHeaders fetches headers
+func (f *IPLDPGFetcher) FetchHeader(tx *sqlx.Tx, c HeaderModel) (ipfs.BlockModel, error) {
+ log.Debug("fetching header ipld")
+ headerBytes, err := shared.FetchIPLD(tx, c.CID)
+ if err != nil {
+ return ipfs.BlockModel{}, err
+ }
+ return ipfs.BlockModel{
+ Data: headerBytes,
+ CID: c.CID,
+ }, nil
+}
+
+// FetchTrxs fetches transactions
+func (f *IPLDPGFetcher) FetchTrxs(tx *sqlx.Tx, cids []TxModel) ([]ipfs.BlockModel, error) {
+ log.Debug("fetching transaction iplds")
+ trxIPLDs := make([]ipfs.BlockModel, len(cids))
+ for i, c := range cids {
+ trxBytes, err := shared.FetchIPLD(tx, c.CID)
+ if err != nil {
+ return nil, err
+ }
+ trxIPLDs[i] = ipfs.BlockModel{
+ Data: trxBytes,
+ CID: c.CID,
+ }
+ }
+ return trxIPLDs, nil
+}
diff --git a/pkg/super_node/btc/publishAndIndexer.go b/pkg/super_node/btc/publishAndIndexer.go
new file mode 100644
index 00000000..1b186b88
--- /dev/null
+++ b/pkg/super_node/btc/publishAndIndexer.go
@@ -0,0 +1,124 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package btc
+
+import (
+ "fmt"
+ "strconv"
+
+ "github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld"
+ "github.com/vulcanize/vulcanizedb/pkg/postgres"
+ "github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+)
+
+// IPLDPublisherAndIndexer satisfies the IPLDPublisher interface for bitcoin
+// It interfaces directly with the public.blocks table of PG-IPFS rather than going through an ipfs intermediary
+// It publishes and indexes IPLDs together in a single sqlx.Tx
+type IPLDPublisherAndIndexer struct {
+ indexer *CIDIndexer
+}
+
+// NewIPLDPublisherAndIndexer creates a pointer to a new IPLDPublisherAndIndexer which satisfies the IPLDPublisher interface
+func NewIPLDPublisherAndIndexer(db *postgres.DB) *IPLDPublisherAndIndexer {
+ return &IPLDPublisherAndIndexer{
+ indexer: NewCIDIndexer(db),
+ }
+}
+
+// Publish publishes an IPLDPayload to IPFS and returns the corresponding CIDPayload
+func (pub *IPLDPublisherAndIndexer) Publish(payload shared.ConvertedData) (shared.CIDsForIndexing, error) {
+ ipldPayload, ok := payload.(ConvertedPayload)
+ if !ok {
+ return nil, fmt.Errorf("btc publisher expected payload type %T got %T", ConvertedPayload{}, payload)
+ }
+ // Generate the iplds
+ headerNode, txNodes, txTrieNodes, err := ipld.FromHeaderAndTxs(ipldPayload.Header, ipldPayload.Txs)
+ if err != nil {
+ return nil, err
+ }
+
+ // Begin new db tx
+ tx, err := pub.indexer.db.Beginx()
+ if err != nil {
+ return nil, err
+ }
+ defer func() {
+ if p := recover(); p != nil {
+ shared.Rollback(tx)
+ panic(p)
+ } else if err != nil {
+ shared.Rollback(tx)
+ } else {
+ err = tx.Commit()
+ }
+ }()
+
+ // Publish trie nodes
+ for _, node := range txTrieNodes {
+ if err := shared.PublishIPLD(tx, node); err != nil {
+ return nil, err
+ }
+ }
+
+ // Publish and index header
+ if err := shared.PublishIPLD(tx, headerNode); err != nil {
+ return nil, err
+ }
+ header := HeaderModel{
+ CID: headerNode.Cid().String(),
+ ParentHash: ipldPayload.Header.PrevBlock.String(),
+ BlockNumber: strconv.Itoa(int(ipldPayload.BlockPayload.BlockHeight)),
+ BlockHash: ipldPayload.Header.BlockHash().String(),
+ Timestamp: ipldPayload.Header.Timestamp.UnixNano(),
+ Bits: ipldPayload.Header.Bits,
+ }
+ headerID, err := pub.indexer.indexHeaderCID(tx, header)
+ if err != nil {
+ return nil, err
+ }
+
+ // Publish and index txs
+ for i, txNode := range txNodes {
+ if err := shared.PublishIPLD(tx, txNode); err != nil {
+ return nil, err
+ }
+ txModel := ipldPayload.TxMetaData[i]
+ txModel.CID = txNode.Cid().String()
+ txID, err := pub.indexer.indexTransactionCID(tx, txModel, headerID)
+ if err != nil {
+ return nil, err
+ }
+ for _, input := range txModel.TxInputs {
+ if err := pub.indexer.indexTxInput(tx, input, txID); err != nil {
+ return nil, err
+ }
+ }
+ for _, output := range txModel.TxOutputs {
+ if err := pub.indexer.indexTxOutput(tx, output, txID); err != nil {
+ return nil, err
+ }
+ }
+ }
+
+ // This IPLDPublisher does both publishing and indexing, we do not need to pass anything forward to the indexer
+ return nil, err
+}
+
+// Index satisfies the shared.CIDIndexer interface
+func (pub *IPLDPublisherAndIndexer) Index(cids shared.CIDsForIndexing) error {
+ return nil
+}
diff --git a/pkg/super_node/btc/publishAndIndexer_test.go b/pkg/super_node/btc/publishAndIndexer_test.go
new file mode 100644
index 00000000..b989bb27
--- /dev/null
+++ b/pkg/super_node/btc/publishAndIndexer_test.go
@@ -0,0 +1,121 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package btc_test
+
+import (
+ "bytes"
+
+ "github.com/ipfs/go-cid"
+ "github.com/ipfs/go-ipfs-blockstore"
+ "github.com/ipfs/go-ipfs-ds-help"
+ "github.com/multiformats/go-multihash"
+ . "github.com/onsi/ginkgo"
+ . "github.com/onsi/gomega"
+ "github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld"
+
+ "github.com/vulcanize/vulcanizedb/pkg/postgres"
+ "github.com/vulcanize/vulcanizedb/pkg/super_node/btc"
+ "github.com/vulcanize/vulcanizedb/pkg/super_node/btc/mocks"
+ "github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+)
+
+var _ = Describe("PublishAndIndexer", func() {
+ var (
+ db *postgres.DB
+ err error
+ repo *btc.IPLDPublisherAndIndexer
+ ipfsPgGet = `SELECT data FROM public.blocks
+ WHERE key = $1`
+ )
+ BeforeEach(func() {
+ db, err = shared.SetupDB()
+ Expect(err).ToNot(HaveOccurred())
+ repo = btc.NewIPLDPublisherAndIndexer(db)
+ })
+ AfterEach(func() {
+ btc.TearDownDB(db)
+ })
+
+ Describe("Publish", func() {
+ It("Published and indexes header and transaction IPLDs in a single tx", func() {
+ emptyReturn, err := repo.Publish(mocks.MockConvertedPayload)
+ Expect(emptyReturn).To(BeNil())
+ Expect(err).ToNot(HaveOccurred())
+ pgStr := `SELECT * FROM btc.header_cids
+ WHERE block_number = $1`
+ // check header was properly indexed
+ buf := bytes.NewBuffer(make([]byte, 0, 80))
+ err = mocks.MockBlock.Header.Serialize(buf)
+ Expect(err).ToNot(HaveOccurred())
+ headerBytes := buf.Bytes()
+ c, _ := ipld.RawdataToCid(ipld.MBitcoinHeader, headerBytes, multihash.DBL_SHA2_256)
+ header := new(btc.HeaderModel)
+ err = db.Get(header, pgStr, mocks.MockHeaderMetaData.BlockNumber)
+ Expect(err).ToNot(HaveOccurred())
+ Expect(header.CID).To(Equal(c.String()))
+ Expect(header.BlockNumber).To(Equal(mocks.MockHeaderMetaData.BlockNumber))
+ Expect(header.Bits).To(Equal(mocks.MockHeaderMetaData.Bits))
+ Expect(header.Timestamp).To(Equal(mocks.MockHeaderMetaData.Timestamp))
+ Expect(header.BlockHash).To(Equal(mocks.MockHeaderMetaData.BlockHash))
+ Expect(header.ParentHash).To(Equal(mocks.MockHeaderMetaData.ParentHash))
+ dc, err := cid.Decode(header.CID)
+ Expect(err).ToNot(HaveOccurred())
+ mhKey := dshelp.CidToDsKey(dc)
+ prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
+ var data []byte
+ err = db.Get(&data, ipfsPgGet, prefixedKey)
+ Expect(err).ToNot(HaveOccurred())
+ Expect(data).To(Equal(headerBytes))
+
+ // check that txs were properly indexed
+ trxs := make([]btc.TxModel, 0)
+ pgStr = `SELECT transaction_cids.id, transaction_cids.header_id, transaction_cids.index,
+ transaction_cids.tx_hash, transaction_cids.cid, transaction_cids.segwit, transaction_cids.witness_hash
+ FROM btc.transaction_cids INNER JOIN btc.header_cids ON (transaction_cids.header_id = header_cids.id)
+ WHERE header_cids.block_number = $1`
+ err = db.Select(&trxs, pgStr, mocks.MockHeaderMetaData.BlockNumber)
+ Expect(err).ToNot(HaveOccurred())
+ Expect(len(trxs)).To(Equal(3))
+ txData := make([][]byte, len(mocks.MockTransactions))
+ txCIDs := make([]string, len(mocks.MockTransactions))
+ for i, m := range mocks.MockTransactions {
+ buf := bytes.NewBuffer(make([]byte, 0))
+ err = m.MsgTx().Serialize(buf)
+ Expect(err).ToNot(HaveOccurred())
+ tx := buf.Bytes()
+ txData[i] = tx
+ c, _ := ipld.RawdataToCid(ipld.MBitcoinTx, tx, multihash.DBL_SHA2_256)
+ txCIDs[i] = c.String()
+ }
+ for _, tx := range trxs {
+ Expect(tx.SegWit).To(Equal(false))
+ Expect(tx.HeaderID).To(Equal(header.ID))
+ Expect(tx.WitnessHash).To(Equal(""))
+ Expect(tx.CID).To(Equal(txCIDs[tx.Index]))
+ Expect(tx.TxHash).To(Equal(mocks.MockBlock.Transactions[tx.Index].TxHash().String()))
+ dc, err := cid.Decode(tx.CID)
+ Expect(err).ToNot(HaveOccurred())
+ mhKey := dshelp.CidToDsKey(dc)
+ prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
+ var data []byte
+ err = db.Get(&data, ipfsPgGet, prefixedKey)
+ Expect(err).ToNot(HaveOccurred())
+ Expect(data).To(Equal(txData[tx.Index]))
+ }
+ })
+ })
+})
diff --git a/pkg/super_node/config.go b/pkg/super_node/config.go
index be3d09c3..2bb83cc7 100644
--- a/pkg/super_node/config.go
+++ b/pkg/super_node/config.go
@@ -52,6 +52,7 @@ type Config struct {
// Ubiquitous fields
Chain shared.ChainType
IPFSPath string
+ IPFSMode shared.IPFSMode
DB *postgres.DB
DBConfig config.Database
Quit chan bool
@@ -98,10 +99,16 @@ func NewSuperNodeConfig() (*Config, error) {
return nil, err
}
- c.IPFSPath, err = shared.GetIPFSPath()
+ c.IPFSMode, err = shared.GetIPFSMode()
if err != nil {
return nil, err
}
+ if c.IPFSMode == shared.LocalInterface || c.IPFSMode == shared.RemoteClient {
+ c.IPFSPath, err = shared.GetIPFSPath()
+ if err != nil {
+ return nil, err
+ }
+ }
c.Sync = viper.GetBool("superNode.sync")
if c.Sync {
diff --git a/pkg/super_node/constructors.go b/pkg/super_node/constructors.go
index fa62fb7d..283d3141 100644
--- a/pkg/super_node/constructors.go
+++ b/pkg/super_node/constructors.go
@@ -44,12 +44,26 @@ func NewResponseFilterer(chain shared.ChainType) (shared.ResponseFilterer, error
}
// NewCIDIndexer constructs a CIDIndexer for the provided chain type
-func NewCIDIndexer(chain shared.ChainType, db *postgres.DB) (shared.CIDIndexer, error) {
+func NewCIDIndexer(chain shared.ChainType, db *postgres.DB, ipfsMode shared.IPFSMode) (shared.CIDIndexer, error) {
switch chain {
case shared.Ethereum:
- return eth.NewCIDIndexer(db), nil
+ switch ipfsMode {
+ case shared.LocalInterface, shared.RemoteClient:
+ return eth.NewCIDIndexer(db), nil
+ case shared.DirectPostgres:
+ return eth.NewIPLDPublisherAndIndexer(db), nil
+ default:
+ return nil, fmt.Errorf("ethereum CIDIndexer unexpected ipfs mode %s", ipfsMode.String())
+ }
case shared.Bitcoin:
- return btc.NewCIDIndexer(db), nil
+ switch ipfsMode {
+ case shared.LocalInterface, shared.RemoteClient:
+ return btc.NewCIDIndexer(db), nil
+ case shared.DirectPostgres:
+ return eth.NewIPLDPublisherAndIndexer(db), nil
+ default:
+ return nil, fmt.Errorf("bitcoin CIDIndexer unexpected ipfs mode %s", ipfsMode.String())
+ }
default:
return nil, fmt.Errorf("invalid chain %s for indexer constructor", chain.String())
}
@@ -122,24 +136,52 @@ func NewPayloadConverter(chain shared.ChainType) (shared.PayloadConverter, error
}
// NewIPLDFetcher constructs an IPLDFetcher for the provided chain type
-func NewIPLDFetcher(chain shared.ChainType, ipfsPath string) (shared.IPLDFetcher, error) {
+func NewIPLDFetcher(chain shared.ChainType, ipfsPath string, db *postgres.DB, ipfsMode shared.IPFSMode) (shared.IPLDFetcher, error) {
switch chain {
case shared.Ethereum:
- return eth.NewIPLDFetcher(ipfsPath)
+ switch ipfsMode {
+ case shared.LocalInterface, shared.RemoteClient:
+ return eth.NewIPLDFetcher(ipfsPath)
+ case shared.DirectPostgres:
+ return eth.NewIPLDPGFetcher(db), nil
+ default:
+ return nil, fmt.Errorf("ethereum IPLDFetcher unexpected ipfs mode %s", ipfsMode.String())
+ }
case shared.Bitcoin:
- return btc.NewIPLDFetcher(ipfsPath)
+ switch ipfsMode {
+ case shared.LocalInterface, shared.RemoteClient:
+ return btc.NewIPLDFetcher(ipfsPath)
+ case shared.DirectPostgres:
+ return btc.NewIPLDPGFetcher(db), nil
+ default:
+ return nil, fmt.Errorf("bitcoin IPLDFetcher unexpected ipfs mode %s", ipfsMode.String())
+ }
default:
return nil, fmt.Errorf("invalid chain %s for IPLD fetcher constructor", chain.String())
}
}
// NewIPLDPublisher constructs an IPLDPublisher for the provided chain type
-func NewIPLDPublisher(chain shared.ChainType, ipfsPath string) (shared.IPLDPublisher, error) {
+func NewIPLDPublisher(chain shared.ChainType, ipfsPath string, db *postgres.DB, ipfsMode shared.IPFSMode) (shared.IPLDPublisher, error) {
switch chain {
case shared.Ethereum:
- return eth.NewIPLDPublisher(ipfsPath)
+ switch ipfsMode {
+ case shared.LocalInterface, shared.RemoteClient:
+ return eth.NewIPLDPublisher(ipfsPath)
+ case shared.DirectPostgres:
+ return eth.NewIPLDPublisherAndIndexer(db), nil
+ default:
+ return nil, fmt.Errorf("ethereum IPLDPublisher unexpected ipfs mode %s", ipfsMode.String())
+ }
case shared.Bitcoin:
- return btc.NewIPLDPublisher(ipfsPath)
+ switch ipfsMode {
+ case shared.LocalInterface, shared.RemoteClient:
+ return btc.NewIPLDPublisher(ipfsPath)
+ case shared.DirectPostgres:
+ return btc.NewIPLDPublisherAndIndexer(db), nil
+ default:
+ return nil, fmt.Errorf("bitcoin IPLDPublisher unexpected ipfs mode %s", ipfsMode.String())
+ }
default:
return nil, fmt.Errorf("invalid chain %s for publisher constructor", chain.String())
}
@@ -149,7 +191,7 @@ func NewIPLDPublisher(chain shared.ChainType, ipfsPath string) (shared.IPLDPubli
func NewPublicAPI(chain shared.ChainType, db *postgres.DB, ipfsPath string) (rpc.API, error) {
switch chain {
case shared.Ethereum:
- backend, err := eth.NewEthBackend(db, ipfsPath)
+ backend, err := eth.NewEthBackend(db)
if err != nil {
return rpc.API{}, err
}
diff --git a/pkg/super_node/eth/api.go b/pkg/super_node/eth/api.go
index f882bbb3..9614d3fb 100644
--- a/pkg/super_node/eth/api.go
+++ b/pkg/super_node/eth/api.go
@@ -20,6 +20,8 @@ import (
"context"
"math/big"
+ "github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+
"github.com/ethereum/go-ethereum"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
@@ -34,19 +36,19 @@ const APIName = "eth"
const APIVersion = "0.0.1"
type PublicEthAPI struct {
- b *Backend
+ B *Backend
}
// NewPublicEthAPI creates a new PublicEthAPI with the provided underlying Backend
func NewPublicEthAPI(b *Backend) *PublicEthAPI {
return &PublicEthAPI{
- b: b,
+ B: b,
}
}
// BlockNumber returns the block number of the chain head.
func (pea *PublicEthAPI) BlockNumber() hexutil.Uint64 {
- number, _ := pea.b.Retriever.RetrieveLastBlockNumber()
+ number, _ := pea.B.Retriever.RetrieveLastBlockNumber()
return hexutil.Uint64(number)
}
@@ -73,23 +75,36 @@ func (pea *PublicEthAPI) GetLogs(ctx context.Context, crit ethereum.FilterQuery)
LogAddresses: addrStrs,
Topics: topicStrSets,
}
- tx, err := pea.b.DB.Beginx()
+
+ // Begin tx
+ tx, err := pea.B.DB.Beginx()
if err != nil {
return nil, err
}
+ defer func() {
+ if p := recover(); p != nil {
+ shared.Rollback(tx)
+ panic(p)
+ } else if err != nil {
+ shared.Rollback(tx)
+ } else {
+ err = tx.Commit()
+ }
+ }()
+
// If we have a blockhash to filter on, fire off single retrieval query
if crit.BlockHash != nil {
- rctCIDs, err := pea.b.Retriever.RetrieveRctCIDs(tx, filter, 0, crit.BlockHash, nil)
+ rctCIDs, err := pea.B.Retriever.RetrieveRctCIDs(tx, filter, 0, crit.BlockHash, nil)
+ if err != nil {
+ return nil, err
+ }
+ rctIPLDs, err := pea.B.Fetcher.FetchRcts(tx, rctCIDs)
if err != nil {
return nil, err
}
if err := tx.Commit(); err != nil {
return nil, err
}
- rctIPLDs, err := pea.b.Fetcher.FetchRcts(rctCIDs)
- if err != nil {
- return nil, err
- }
return extractLogsOfInterest(rctIPLDs, filter.Topics)
}
// Otherwise, create block range from criteria
@@ -97,14 +112,14 @@ func (pea *PublicEthAPI) GetLogs(ctx context.Context, crit ethereum.FilterQuery)
startingBlock := crit.FromBlock
endingBlock := crit.ToBlock
if startingBlock == nil {
- startingBlockInt, err := pea.b.Retriever.RetrieveFirstBlockNumber()
+ startingBlockInt, err := pea.B.Retriever.RetrieveFirstBlockNumber()
if err != nil {
return nil, err
}
startingBlock = big.NewInt(startingBlockInt)
}
if endingBlock == nil {
- endingBlockInt, err := pea.b.Retriever.RetrieveLastBlockNumber()
+ endingBlockInt, err := pea.B.Retriever.RetrieveLastBlockNumber()
if err != nil {
return nil, err
}
@@ -114,27 +129,28 @@ func (pea *PublicEthAPI) GetLogs(ctx context.Context, crit ethereum.FilterQuery)
end := endingBlock.Int64()
allRctCIDs := make([]ReceiptModel, 0)
for i := start; i <= end; i++ {
- rctCIDs, err := pea.b.Retriever.RetrieveRctCIDs(tx, filter, i, nil, nil)
+ rctCIDs, err := pea.B.Retriever.RetrieveRctCIDs(tx, filter, i, nil, nil)
if err != nil {
return nil, err
}
allRctCIDs = append(allRctCIDs, rctCIDs...)
}
- if err := tx.Commit(); err != nil {
- return nil, err
- }
- rctIPLDs, err := pea.b.Fetcher.FetchRcts(allRctCIDs)
+ rctIPLDs, err := pea.B.Fetcher.FetchRcts(tx, allRctCIDs)
if err != nil {
return nil, err
}
- return extractLogsOfInterest(rctIPLDs, filter.Topics)
+ if err := tx.Commit(); err != nil {
+ return nil, err
+ }
+ logs, err := extractLogsOfInterest(rctIPLDs, filter.Topics)
+ return logs, err // need to return err variable so that we return the err = tx.Commit() assignment in the defer
}
// GetHeaderByNumber returns the requested canonical block header.
// * When blockNr is -1 the chain head is returned.
// * We cannot support pending block calls since we do not have an active miner
func (pea *PublicEthAPI) GetHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (map[string]interface{}, error) {
- header, err := pea.b.HeaderByNumber(ctx, number)
+ header, err := pea.B.HeaderByNumber(ctx, number)
if header != nil && err == nil {
return pea.rpcMarshalHeader(header)
}
@@ -147,7 +163,7 @@ func (pea *PublicEthAPI) GetHeaderByNumber(ctx context.Context, number rpc.Block
// * When fullTx is true all transactions in the block are returned, otherwise
// only the transaction hash is returned.
func (pea *PublicEthAPI) GetBlockByNumber(ctx context.Context, number rpc.BlockNumber, fullTx bool) (map[string]interface{}, error) {
- block, err := pea.b.BlockByNumber(ctx, number)
+ block, err := pea.B.BlockByNumber(ctx, number)
if block != nil && err == nil {
return pea.rpcMarshalBlock(block, true, fullTx)
}
@@ -157,7 +173,7 @@ func (pea *PublicEthAPI) GetBlockByNumber(ctx context.Context, number rpc.BlockN
// GetBlockByHash returns the requested block. When fullTx is true all transactions in the block are returned in full
// detail, otherwise only the transaction hash is returned.
func (pea *PublicEthAPI) GetBlockByHash(ctx context.Context, hash common.Hash, fullTx bool) (map[string]interface{}, error) {
- block, err := pea.b.BlockByHash(ctx, hash)
+ block, err := pea.B.BlockByHash(ctx, hash)
if block != nil {
return pea.rpcMarshalBlock(block, true, fullTx)
}
@@ -168,7 +184,7 @@ func (pea *PublicEthAPI) GetBlockByHash(ctx context.Context, hash common.Hash, f
// SuperNode cannot currently handle pending/tx_pool txs
func (pea *PublicEthAPI) GetTransactionByHash(ctx context.Context, hash common.Hash) (*RPCTransaction, error) {
// Try to return an already finalized transaction
- tx, blockHash, blockNumber, index, err := pea.b.GetTransaction(ctx, hash)
+ tx, blockHash, blockNumber, index, err := pea.B.GetTransaction(ctx, hash)
if err != nil {
return nil, err
}
diff --git a/pkg/super_node/eth/api_test.go b/pkg/super_node/eth/api_test.go
index 60106aee..0c016486 100644
--- a/pkg/super_node/eth/api_test.go
+++ b/pkg/super_node/eth/api_test.go
@@ -27,12 +27,9 @@ import (
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/rpc"
- "github.com/ipfs/go-block-format"
- "github.com/ipfs/go-cid"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
- mocks3 "github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks"
"github.com/vulcanize/vulcanizedb/pkg/postgres"
"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
"github.com/vulcanize/vulcanizedb/pkg/super_node/eth/mocks"
@@ -85,44 +82,27 @@ var (
var _ = Describe("API", func() {
var (
- db *postgres.DB
- retriever *eth.CIDRetriever
- fetcher *eth.IPLDFetcher
- indexer *eth.CIDIndexer
- backend *eth.Backend
- api *eth.PublicEthAPI
+ db *postgres.DB
+ retriever *eth.CIDRetriever
+ fetcher *eth.IPLDPGFetcher
+ indexAndPublisher *eth.IPLDPublisherAndIndexer
+ backend *eth.Backend
+ api *eth.PublicEthAPI
)
BeforeEach(func() {
var err error
db, err = shared.SetupDB()
Expect(err).ToNot(HaveOccurred())
retriever = eth.NewCIDRetriever(db)
- blocksToReturn := map[cid.Cid]blocks.Block{
- mocks.HeaderCID: mocks.HeaderIPLD,
- mocks.Trx1CID: mocks.Trx1IPLD,
- mocks.Trx2CID: mocks.Trx2IPLD,
- mocks.Trx3CID: mocks.Trx3IPLD,
- mocks.Rct1CID: mocks.Rct1IPLD,
- mocks.Rct2CID: mocks.Rct2IPLD,
- mocks.Rct3CID: mocks.Rct3IPLD,
- mocks.State1CID: mocks.State1IPLD,
- mocks.State2CID: mocks.State2IPLD,
- mocks.StorageCID: mocks.StorageIPLD,
- }
- mockBlockService := &mocks3.MockIPFSBlockService{
- Blocks: blocksToReturn,
- }
- fetcher = ð.IPLDFetcher{
- BlockService: mockBlockService,
- }
- indexer = eth.NewCIDIndexer(db)
+ fetcher = eth.NewIPLDPGFetcher(db)
+ indexAndPublisher = eth.NewIPLDPublisherAndIndexer(db)
backend = ð.Backend{
Retriever: retriever,
Fetcher: fetcher,
DB: db,
}
api = eth.NewPublicEthAPI(backend)
- err = indexer.Index(mocks.MockCIDPayload)
+ _, err = indexAndPublisher.Publish(mocks.MockConvertedPayload)
Expect(err).ToNot(HaveOccurred())
uncles := mocks.MockBlock.Uncles()
uncleHashes := make([]common.Hash, len(uncles))
@@ -186,8 +166,20 @@ var _ = Describe("API", func() {
number, err := strconv.ParseInt(mocks.MockCIDPayload.HeaderCID.BlockNumber, 10, 64)
Expect(err).ToNot(HaveOccurred())
header, err := api.GetHeaderByNumber(context.Background(), rpc.BlockNumber(number))
+ Expect(err).ToNot(HaveOccurred())
Expect(header).To(Equal(expectedHeader))
})
+
+ It("Throws an error if a header cannot be found", func() {
+ number, err := strconv.ParseInt(mocks.MockCIDPayload.HeaderCID.BlockNumber, 10, 64)
+ Expect(err).ToNot(HaveOccurred())
+ header, err := api.GetHeaderByNumber(context.Background(), rpc.BlockNumber(number+1))
+ Expect(err).To(HaveOccurred())
+ Expect(err.Error()).To(ContainSubstring("header at block %d is not available", number+1))
+ Expect(header).To(BeNil())
+ _, err = api.B.DB.Beginx()
+ Expect(err).ToNot(HaveOccurred())
+ })
})
Describe("GetBlockByHash", func() {
diff --git a/pkg/super_node/eth/backend.go b/pkg/super_node/eth/backend.go
index 1365c47c..fd6ce01d 100644
--- a/pkg/super_node/eth/backend.go
+++ b/pkg/super_node/eth/backend.go
@@ -22,13 +22,13 @@ import (
"fmt"
"math/big"
+ "github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/rpc"
- "github.com/sirupsen/logrus"
-
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"github.com/vulcanize/vulcanizedb/pkg/postgres"
)
@@ -39,26 +39,21 @@ var (
type Backend struct {
Retriever *CIDRetriever
- Fetcher *IPLDFetcher
+ Fetcher *IPLDPGFetcher
DB *postgres.DB
}
-func NewEthBackend(db *postgres.DB, ipfsPath string) (*Backend, error) {
+func NewEthBackend(db *postgres.DB) (*Backend, error) {
r := NewCIDRetriever(db)
- f, err := NewIPLDFetcher(ipfsPath)
- if err != nil {
- return nil, err
- }
return &Backend{
Retriever: r,
- Fetcher: f,
DB: db,
}, nil
}
func (b *Backend) HeaderByNumber(ctx context.Context, blockNumber rpc.BlockNumber) (*types.Header, error) {
- number := blockNumber.Int64()
var err error
+ number := blockNumber.Int64()
if blockNumber == rpc.LatestBlockNumber {
number, err = b.Retriever.RetrieveLastBlockNumber()
if err != nil {
@@ -68,19 +63,26 @@ func (b *Backend) HeaderByNumber(ctx context.Context, blockNumber rpc.BlockNumbe
if blockNumber == rpc.PendingBlockNumber {
return nil, errPendingBlockNumber
}
- // Retrieve the CIDs for headers at this height
+
+ // Begin tx
tx, err := b.DB.Beginx()
if err != nil {
return nil, err
}
+ defer func() {
+ if p := recover(); p != nil {
+ shared.Rollback(tx)
+ panic(p)
+ } else if err != nil {
+ shared.Rollback(tx)
+ } else {
+ err = tx.Commit()
+ }
+ }()
+
+ // Retrieve the CIDs for headers at this height
headerCids, err := b.Retriever.RetrieveHeaderCIDs(tx, number)
if err != nil {
- if err := tx.Rollback(); err != nil {
- logrus.Error(err)
- }
- return nil, err
- }
- if err := tx.Commit(); err != nil {
return nil, err
}
// If there are none, throw an error
@@ -88,7 +90,7 @@ func (b *Backend) HeaderByNumber(ctx context.Context, blockNumber rpc.BlockNumbe
return nil, fmt.Errorf("header at block %d is not available", number)
}
// Fetch the header IPLDs for those CIDs
- headerIPLD, err := b.Fetcher.FetchHeader(headerCids[0])
+ headerIPLD, err := b.Fetcher.FetchHeader(tx, headerCids[0])
if err != nil {
return nil, err
}
@@ -96,10 +98,8 @@ func (b *Backend) HeaderByNumber(ctx context.Context, blockNumber rpc.BlockNumbe
// We throw an error in FetchHeaders() if the number of headers does not match the number of CIDs and we already
// confirmed the number of CIDs is greater than 0 so there is no need to bound check the slice before accessing
var header types.Header
- if err := rlp.DecodeBytes(headerIPLD.Data, &header); err != nil {
- return nil, err
- }
- return &header, nil
+ err = rlp.DecodeBytes(headerIPLD.Data, &header)
+ return &header, err
}
// GetTd retrieves and returns the total difficulty at the given block hash
@@ -120,24 +120,29 @@ func (b *Backend) GetTd(blockHash common.Hash) (*big.Int, error) {
// GetLogs returns all the logs for the given block hash
func (b *Backend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) {
+ // Begin tx
tx, err := b.DB.Beginx()
if err != nil {
return nil, err
}
+ defer func() {
+ if p := recover(); p != nil {
+ shared.Rollback(tx)
+ panic(p)
+ } else if err != nil {
+ shared.Rollback(tx)
+ } else {
+ err = tx.Commit()
+ }
+ }()
receiptCIDs, err := b.Retriever.RetrieveRctCIDs(tx, ReceiptFilter{}, 0, &hash, nil)
if err != nil {
- if err := tx.Rollback(); err != nil {
- logrus.Error(err)
- }
- return nil, err
- }
- if err := tx.Commit(); err != nil {
return nil, err
}
if len(receiptCIDs) == 0 {
return nil, nil
}
- receiptIPLDs, err := b.Fetcher.FetchRcts(receiptCIDs)
+ receiptIPLDs, err := b.Fetcher.FetchRcts(tx, receiptCIDs)
if err != nil {
return nil, err
}
@@ -149,15 +154,15 @@ func (b *Backend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log
}
logs[i] = rct.Logs
}
- return logs, nil
+ return logs, err
}
// BlockByNumber returns the requested canonical block.
// Since the SuperNode can contain forked blocks, it is recommended to fetch BlockByHash as
// fetching by number can return non-deterministic results (returns the first block found at that height)
func (b *Backend) BlockByNumber(ctx context.Context, blockNumber rpc.BlockNumber) (*types.Block, error) {
- number := blockNumber.Int64()
var err error
+ number := blockNumber.Int64()
if blockNumber == rpc.LatestBlockNumber {
number, err = b.Retriever.RetrieveLastBlockNumber()
if err != nil {
@@ -173,8 +178,24 @@ func (b *Backend) BlockByNumber(ctx context.Context, blockNumber rpc.BlockNumber
return nil, err
}
+ // Begin tx
+ tx, err := b.DB.Beginx()
+ if err != nil {
+ return nil, err
+ }
+ defer func() {
+ if p := recover(); p != nil {
+ shared.Rollback(tx)
+ panic(p)
+ } else if err != nil {
+ shared.Rollback(tx)
+ } else {
+ err = tx.Commit()
+ }
+ }()
+
// Fetch and decode the header IPLD
- headerIPLD, err := b.Fetcher.FetchHeader(headerCID)
+ headerIPLD, err := b.Fetcher.FetchHeader(tx, headerCID)
if err != nil {
return nil, err
}
@@ -183,7 +204,7 @@ func (b *Backend) BlockByNumber(ctx context.Context, blockNumber rpc.BlockNumber
return nil, err
}
// Fetch and decode the uncle IPLDs
- uncleIPLDs, err := b.Fetcher.FetchUncles(uncleCIDs)
+ uncleIPLDs, err := b.Fetcher.FetchUncles(tx, uncleCIDs)
if err != nil {
return nil, err
}
@@ -196,20 +217,20 @@ func (b *Backend) BlockByNumber(ctx context.Context, blockNumber rpc.BlockNumber
uncles = append(uncles, &uncle)
}
// Fetch and decode the transaction IPLDs
- txIPLDs, err := b.Fetcher.FetchTrxs(txCIDs)
+ txIPLDs, err := b.Fetcher.FetchTrxs(tx, txCIDs)
if err != nil {
return nil, err
}
var transactions []*types.Transaction
for _, txIPLD := range txIPLDs {
- var tx types.Transaction
- if err := rlp.DecodeBytes(txIPLD.Data, &tx); err != nil {
+ var transaction types.Transaction
+ if err := rlp.DecodeBytes(txIPLD.Data, &transaction); err != nil {
return nil, err
}
- transactions = append(transactions, &tx)
+ transactions = append(transactions, &transaction)
}
// Fetch and decode the receipt IPLDs
- rctIPLDs, err := b.Fetcher.FetchRcts(rctCIDs)
+ rctIPLDs, err := b.Fetcher.FetchRcts(tx, rctCIDs)
if err != nil {
return nil, err
}
@@ -222,7 +243,7 @@ func (b *Backend) BlockByNumber(ctx context.Context, blockNumber rpc.BlockNumber
receipts = append(receipts, &receipt)
}
// Compose everything together into a complete block
- return types.NewBlock(&header, transactions, uncles, receipts), nil
+ return types.NewBlock(&header, transactions, uncles, receipts), err
}
// BlockByHash returns the requested block. When fullTx is true all transactions in the block are returned in full
@@ -233,8 +254,25 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo
if err != nil {
return nil, err
}
+
+ // Begin tx
+ tx, err := b.DB.Beginx()
+ if err != nil {
+ return nil, err
+ }
+ defer func() {
+ if p := recover(); p != nil {
+ shared.Rollback(tx)
+ panic(p)
+ } else if err != nil {
+ shared.Rollback(tx)
+ } else {
+ err = tx.Commit()
+ }
+ }()
+
// Fetch and decode the header IPLD
- headerIPLD, err := b.Fetcher.FetchHeader(headerCID)
+ headerIPLD, err := b.Fetcher.FetchHeader(tx, headerCID)
if err != nil {
return nil, err
}
@@ -243,7 +281,7 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo
return nil, err
}
// Fetch and decode the uncle IPLDs
- uncleIPLDs, err := b.Fetcher.FetchUncles(uncleCIDs)
+ uncleIPLDs, err := b.Fetcher.FetchUncles(tx, uncleCIDs)
if err != nil {
return nil, err
}
@@ -256,20 +294,20 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo
uncles = append(uncles, &uncle)
}
// Fetch and decode the transaction IPLDs
- txIPLDs, err := b.Fetcher.FetchTrxs(txCIDs)
+ txIPLDs, err := b.Fetcher.FetchTrxs(tx, txCIDs)
if err != nil {
return nil, err
}
var transactions []*types.Transaction
for _, txIPLD := range txIPLDs {
- var tx types.Transaction
- if err := rlp.DecodeBytes(txIPLD.Data, &tx); err != nil {
+ var transaction types.Transaction
+ if err := rlp.DecodeBytes(txIPLD.Data, &transaction); err != nil {
return nil, err
}
- transactions = append(transactions, &tx)
+ transactions = append(transactions, &transaction)
}
// Fetch and decode the receipt IPLDs
- rctIPLDs, err := b.Fetcher.FetchRcts(rctCIDs)
+ rctIPLDs, err := b.Fetcher.FetchRcts(tx, rctCIDs)
if err != nil {
return nil, err
}
@@ -282,7 +320,7 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo
receipts = append(receipts, &receipt)
}
// Compose everything together into a complete block
- return types.NewBlock(&header, transactions, uncles, receipts), nil
+ return types.NewBlock(&header, transactions, uncles, receipts), err
}
// GetTransaction retrieves a tx by hash
@@ -301,15 +339,35 @@ func (b *Backend) GetTransaction(ctx context.Context, txHash common.Hash) (*type
if err := b.DB.Get(&txCIDWithHeaderInfo, pgStr, txHash.String()); err != nil {
return nil, common.Hash{}, 0, 0, err
}
- txIPLD, err := b.Fetcher.FetchTrxs([]TxModel{{CID: txCIDWithHeaderInfo.CID}})
+
+ // Begin tx
+ tx, err := b.DB.Beginx()
if err != nil {
return nil, common.Hash{}, 0, 0, err
}
+ defer func() {
+ if p := recover(); p != nil {
+ shared.Rollback(tx)
+ panic(p)
+ } else if err != nil {
+ shared.Rollback(tx)
+ } else {
+ err = tx.Commit()
+ }
+ }()
+
+ txIPLD, err := b.Fetcher.FetchTrxs(tx, []TxModel{{CID: txCIDWithHeaderInfo.CID}})
+ if err != nil {
+ return nil, common.Hash{}, 0, 0, err
+ }
+ if err := tx.Commit(); err != nil {
+ return nil, common.Hash{}, 0, 0, err
+ }
var transaction types.Transaction
if err := rlp.DecodeBytes(txIPLD[0].Data, &transaction); err != nil {
return nil, common.Hash{}, 0, 0, err
}
- return &transaction, common.HexToHash(txCIDWithHeaderInfo.BlockHash), uint64(txCIDWithHeaderInfo.BlockNumber), uint64(txCIDWithHeaderInfo.Index), nil
+ return &transaction, common.HexToHash(txCIDWithHeaderInfo.BlockHash), uint64(txCIDWithHeaderInfo.BlockNumber), uint64(txCIDWithHeaderInfo.Index), err
}
// extractLogsOfInterest returns logs from the receipt IPLD
@@ -364,7 +422,7 @@ func sliceContainsHash(slice []string, hash common.Hash) int {
// a `PublicEthAPI`.
func (pea *PublicEthAPI) rpcMarshalHeader(header *types.Header) (map[string]interface{}, error) {
fields := RPCMarshalHeader(header)
- td, err := pea.b.GetTd(header.Hash())
+ td, err := pea.B.GetTd(header.Hash())
if err != nil {
return nil, err
}
@@ -403,7 +461,7 @@ func (pea *PublicEthAPI) rpcMarshalBlock(b *types.Block, inclTx bool, fullTx boo
if err != nil {
return nil, err
}
- td, err := pea.b.GetTd(b.Hash())
+ td, err := pea.B.GetTd(b.Hash())
if err != nil {
return nil, err
}
diff --git a/pkg/super_node/eth/retriever.go b/pkg/super_node/eth/cid_retriever.go
similarity index 94%
rename from pkg/super_node/eth/retriever.go
rename to pkg/super_node/eth/cid_retriever.go
index 539052db..e7acf571 100644
--- a/pkg/super_node/eth/retriever.go
+++ b/pkg/super_node/eth/cid_retriever.go
@@ -17,17 +17,17 @@
package eth
import (
+ "database/sql"
"fmt"
"math/big"
- "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
-
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
"github.com/jmoiron/sqlx"
"github.com/lib/pq"
log "github.com/sirupsen/logrus"
+ utils "github.com/vulcanize/vulcanizedb/libraries/shared/utilities"
"github.com/vulcanize/vulcanizedb/pkg/postgres"
"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
)
@@ -65,17 +65,26 @@ func (ecr *CIDRetriever) Retrieve(filter shared.SubscriptionSettings, blockNumbe
return nil, true, fmt.Errorf("eth retriever expected filter type %T got %T", &SubscriptionSettings{}, filter)
}
log.Debug("retrieving cids")
+
+ // Begin new db tx
tx, err := ecr.db.Beginx()
if err != nil {
return nil, true, err
}
+ defer func() {
+ if p := recover(); p != nil {
+ shared.Rollback(tx)
+ panic(p)
+ } else if err != nil {
+ shared.Rollback(tx)
+ } else {
+ err = tx.Commit()
+ }
+ }()
// Retrieve cached header CIDs at this block height
headers, err := ecr.RetrieveHeaderCIDs(tx, blockNumber)
if err != nil {
- if err := tx.Rollback(); err != nil {
- log.Error(err)
- }
log.Error("header cid retrieval error")
return nil, true, err
}
@@ -91,9 +100,6 @@ func (ecr *CIDRetriever) Retrieve(filter shared.SubscriptionSettings, blockNumbe
// Retrieve uncle cids for this header id
uncleCIDs, err := ecr.RetrieveUncleCIDsByHeaderID(tx, header.ID)
if err != nil {
- if err := tx.Rollback(); err != nil {
- log.Error(err)
- }
log.Error("uncle cid retrieval error")
return nil, true, err
}
@@ -104,9 +110,6 @@ func (ecr *CIDRetriever) Retrieve(filter shared.SubscriptionSettings, blockNumbe
if !streamFilter.TxFilter.Off {
cw.Transactions, err = ecr.RetrieveTxCIDs(tx, streamFilter.TxFilter, header.ID)
if err != nil {
- if err := tx.Rollback(); err != nil {
- log.Error(err)
- }
log.Error("transaction cid retrieval error")
return nil, true, err
}
@@ -122,9 +125,6 @@ func (ecr *CIDRetriever) Retrieve(filter shared.SubscriptionSettings, blockNumbe
if !streamFilter.ReceiptFilter.Off {
cw.Receipts, err = ecr.RetrieveRctCIDsByHeaderID(tx, streamFilter.ReceiptFilter, header.ID, trxIds)
if err != nil {
- if err := tx.Rollback(); err != nil {
- log.Error(err)
- }
log.Error("receipt cid retrieval error")
return nil, true, err
}
@@ -136,9 +136,6 @@ func (ecr *CIDRetriever) Retrieve(filter shared.SubscriptionSettings, blockNumbe
if !streamFilter.StateFilter.Off {
cw.StateNodes, err = ecr.RetrieveStateCIDs(tx, streamFilter.StateFilter, header.ID)
if err != nil {
- if err := tx.Rollback(); err != nil {
- log.Error(err)
- }
log.Error("state cid retrieval error")
return nil, true, err
}
@@ -150,9 +147,6 @@ func (ecr *CIDRetriever) Retrieve(filter shared.SubscriptionSettings, blockNumbe
if !streamFilter.StorageFilter.Off {
cw.StorageNodes, err = ecr.RetrieveStorageCIDs(tx, streamFilter.StorageFilter, header.ID)
if err != nil {
- if err := tx.Rollback(); err != nil {
- log.Error(err)
- }
log.Error("storage cid retrieval error")
return nil, true, err
}
@@ -163,7 +157,7 @@ func (ecr *CIDRetriever) Retrieve(filter shared.SubscriptionSettings, blockNumbe
cws[i] = cw
}
- return cws, empty, tx.Commit()
+ return cws, empty, err
}
// RetrieveHeaderCIDs retrieves and returns all of the header cids at the provided blockheight
@@ -458,7 +452,7 @@ func (ecr *CIDRetriever) RetrieveGapsInData(validationLevel int) ([]shared.Gap,
Start uint64 `db:"start"`
Stop uint64 `db:"stop"`
}, 0)
- if err := ecr.db.Select(&results, pgStr); err != nil {
+ if err := ecr.db.Select(&results, pgStr); err != nil && err != sql.ErrNoRows {
return nil, err
}
emptyGaps := make([]shared.Gap, len(results))
@@ -475,43 +469,44 @@ func (ecr *CIDRetriever) RetrieveGapsInData(validationLevel int) ([]shared.Gap,
WHERE times_validated < $1
ORDER BY block_number`
var heights []uint64
- if err := ecr.db.Select(&heights, pgStr, validationLevel); err != nil {
+ if err := ecr.db.Select(&heights, pgStr, validationLevel); err != nil && err != sql.ErrNoRows {
return nil, err
}
- if len(heights) == 0 {
- return emptyGaps, nil
- }
return append(emptyGaps, utils.MissingHeightsToGaps(heights)...), nil
}
// RetrieveBlockByHash returns all of the CIDs needed to compose an entire block, for a given block hash
func (ecr *CIDRetriever) RetrieveBlockByHash(blockHash common.Hash) (HeaderModel, []UncleModel, []TxModel, []ReceiptModel, error) {
log.Debug("retrieving block cids for block hash ", blockHash.String())
+
+ // Begin new db tx
tx, err := ecr.db.Beginx()
if err != nil {
return HeaderModel{}, nil, nil, nil, err
}
+ defer func() {
+ if p := recover(); p != nil {
+ shared.Rollback(tx)
+ panic(p)
+ } else if err != nil {
+ shared.Rollback(tx)
+ } else {
+ err = tx.Commit()
+ }
+ }()
+
headerCID, err := ecr.RetrieveHeaderCIDByHash(tx, blockHash)
if err != nil {
- if err := tx.Rollback(); err != nil {
- log.Error(err)
- }
log.Error("header cid retrieval error")
return HeaderModel{}, nil, nil, nil, err
}
uncleCIDs, err := ecr.RetrieveUncleCIDsByHeaderID(tx, headerCID.ID)
if err != nil {
- if err := tx.Rollback(); err != nil {
- log.Error(err)
- }
log.Error("uncle cid retrieval error")
return HeaderModel{}, nil, nil, nil, err
}
txCIDs, err := ecr.RetrieveTxCIDsByHeaderID(tx, headerCID.ID)
if err != nil {
- if err := tx.Rollback(); err != nil {
- log.Error(err)
- }
log.Error("tx cid retrieval error")
return HeaderModel{}, nil, nil, nil, err
}
@@ -521,27 +516,33 @@ func (ecr *CIDRetriever) RetrieveBlockByHash(blockHash common.Hash) (HeaderModel
}
rctCIDs, err := ecr.RetrieveReceiptCIDsByTxIDs(tx, txIDs)
if err != nil {
- if err := tx.Rollback(); err != nil {
- log.Error(err)
- }
log.Error("rct cid retrieval error")
- return HeaderModel{}, nil, nil, nil, err
}
- return headerCID, uncleCIDs, txCIDs, rctCIDs, tx.Commit()
+ return headerCID, uncleCIDs, txCIDs, rctCIDs, err
}
// RetrieveBlockByNumber returns all of the CIDs needed to compose an entire block, for a given block number
func (ecr *CIDRetriever) RetrieveBlockByNumber(blockNumber int64) (HeaderModel, []UncleModel, []TxModel, []ReceiptModel, error) {
log.Debug("retrieving block cids for block number ", blockNumber)
+
+ // Begin new db tx
tx, err := ecr.db.Beginx()
if err != nil {
return HeaderModel{}, nil, nil, nil, err
}
+ defer func() {
+ if p := recover(); p != nil {
+ shared.Rollback(tx)
+ panic(p)
+ } else if err != nil {
+ shared.Rollback(tx)
+ } else {
+ err = tx.Commit()
+ }
+ }()
+
headerCID, err := ecr.RetrieveHeaderCIDs(tx, blockNumber)
if err != nil {
- if err := tx.Rollback(); err != nil {
- log.Error(err)
- }
log.Error("header cid retrieval error")
return HeaderModel{}, nil, nil, nil, err
}
@@ -550,17 +551,11 @@ func (ecr *CIDRetriever) RetrieveBlockByNumber(blockNumber int64) (HeaderModel,
}
uncleCIDs, err := ecr.RetrieveUncleCIDsByHeaderID(tx, headerCID[0].ID)
if err != nil {
- if err := tx.Rollback(); err != nil {
- log.Error(err)
- }
log.Error("uncle cid retrieval error")
return HeaderModel{}, nil, nil, nil, err
}
txCIDs, err := ecr.RetrieveTxCIDsByHeaderID(tx, headerCID[0].ID)
if err != nil {
- if err := tx.Rollback(); err != nil {
- log.Error(err)
- }
log.Error("tx cid retrieval error")
return HeaderModel{}, nil, nil, nil, err
}
@@ -570,13 +565,9 @@ func (ecr *CIDRetriever) RetrieveBlockByNumber(blockNumber int64) (HeaderModel,
}
rctCIDs, err := ecr.RetrieveReceiptCIDsByTxIDs(tx, txIDs)
if err != nil {
- if err := tx.Rollback(); err != nil {
- log.Error(err)
- }
log.Error("rct cid retrieval error")
- return HeaderModel{}, nil, nil, nil, err
}
- return headerCID[0], uncleCIDs, txCIDs, rctCIDs, tx.Commit()
+ return headerCID[0], uncleCIDs, txCIDs, rctCIDs, err
}
// RetrieveHeaderCIDByHash returns the header for the given block hash
diff --git a/pkg/super_node/eth/retriever_test.go b/pkg/super_node/eth/cid_retriever_test.go
similarity index 100%
rename from pkg/super_node/eth/retriever_test.go
rename to pkg/super_node/eth/cid_retriever_test.go
diff --git a/pkg/super_node/eth/cleaner.go b/pkg/super_node/eth/cleaner.go
index 37ebcba6..9ef11854 100644
--- a/pkg/super_node/eth/cleaner.go
+++ b/pkg/super_node/eth/cleaner.go
@@ -50,9 +50,7 @@ func (c *Cleaner) ResetValidation(rngs [][2]uint64) error {
SET times_validated = 0
WHERE block_number BETWEEN $1 AND $2`
if _, err := tx.Exec(pgStr, rng[0], rng[1]); err != nil {
- if err := tx.Rollback(); err != nil {
- logrus.Error(err)
- }
+ shared.Rollback(tx)
return err
}
}
@@ -68,9 +66,7 @@ func (c *Cleaner) Clean(rngs [][2]uint64, t shared.DataType) error {
for _, rng := range rngs {
logrus.Infof("eth db cleaner cleaning up block range %d to %d", rng[0], rng[1])
if err := c.clean(tx, rng, t); err != nil {
- if err := tx.Rollback(); err != nil {
- logrus.Error(err)
- }
+ shared.Rollback(tx)
return err
}
}
diff --git a/pkg/super_node/eth/indexer.go b/pkg/super_node/eth/indexer.go
index f9723fe7..0bf179c2 100644
--- a/pkg/super_node/eth/indexer.go
+++ b/pkg/super_node/eth/indexer.go
@@ -50,51 +50,52 @@ func (in *CIDIndexer) Index(cids shared.CIDsForIndexing) error {
if !ok {
return fmt.Errorf("eth indexer expected cids type %T got %T", &CIDPayload{}, cids)
}
+
+ // Begin new db tx
tx, err := in.db.Beginx()
if err != nil {
return err
}
- headerID, err := in.indexHeaderCID(tx, cidPayload.HeaderCID, in.db.NodeID)
- if err != nil {
- if err := tx.Rollback(); err != nil {
- log.Error(err)
+ defer func() {
+ if p := recover(); p != nil {
+ shared.Rollback(tx)
+ panic(p)
+ } else if err != nil {
+ shared.Rollback(tx)
+ } else {
+ err = tx.Commit()
}
+ }()
+
+ headerID, err := in.indexHeaderCID(tx, cidPayload.HeaderCID)
+ if err != nil {
log.Error("eth indexer error when indexing header")
return err
}
for _, uncle := range cidPayload.UncleCIDs {
if err := in.indexUncleCID(tx, uncle, headerID); err != nil {
- if err := tx.Rollback(); err != nil {
- log.Error(err)
- }
log.Error("eth indexer error when indexing uncle")
return err
}
}
if err := in.indexTransactionAndReceiptCIDs(tx, cidPayload, headerID); err != nil {
- if err := tx.Rollback(); err != nil {
- log.Error(err)
- }
log.Error("eth indexer error when indexing transactions and receipts")
return err
}
- if err := in.indexStateAndStorageCIDs(tx, cidPayload, headerID); err != nil {
- if err := tx.Rollback(); err != nil {
- log.Error(err)
- }
+ err = in.indexStateAndStorageCIDs(tx, cidPayload, headerID)
+ if err != nil {
log.Error("eth indexer error when indexing state and storage nodes")
- return err
}
- return tx.Commit()
+ return err
}
-func (in *CIDIndexer) indexHeaderCID(tx *sqlx.Tx, header HeaderModel, nodeID int64) (int64, error) {
+func (in *CIDIndexer) indexHeaderCID(tx *sqlx.Tx, header HeaderModel) (int64, error) {
var headerID int64
err := tx.QueryRowx(`INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, times_validated)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)
ON CONFLICT (block_number, block_hash) DO UPDATE SET (parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, times_validated) = ($3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, eth.header_cids.times_validated + 1)
RETURNING id`,
- header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, header.TotalDifficulty, nodeID, header.Reward, header.StateRoot, header.TxRoot,
+ header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, header.TotalDifficulty, in.db.NodeID, header.Reward, header.StateRoot, header.TxRoot,
header.RctRoot, header.UncleRoot, header.Bloom, header.Timestamp, 1).Scan(&headerID)
return headerID, err
}
@@ -126,6 +127,15 @@ func (in *CIDIndexer) indexTransactionAndReceiptCIDs(tx *sqlx.Tx, payload *CIDPa
return nil
}
+func (in *CIDIndexer) indexTransactionCID(tx *sqlx.Tx, transaction TxModel, headerID int64) (int64, error) {
+ var txID int64
+ err := tx.QueryRowx(`INSERT INTO eth.transaction_cids (header_id, tx_hash, cid, dst, src, index) VALUES ($1, $2, $3, $4, $5, $6)
+ ON CONFLICT (header_id, tx_hash) DO UPDATE SET (cid, dst, src, index) = ($3, $4, $5, $6)
+ RETURNING id`,
+ headerID, transaction.TxHash, transaction.CID, transaction.Dst, transaction.Src, transaction.Index).Scan(&txID)
+ return txID, err
+}
+
func (in *CIDIndexer) indexReceiptCID(tx *sqlx.Tx, cidMeta ReceiptModel, txID int64) error {
_, err := tx.Exec(`INSERT INTO eth.receipt_cids (tx_id, cid, contract, contract_hash, topic0s, topic1s, topic2s, topic3s, log_contracts) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
ON CONFLICT (tx_id) DO UPDATE SET (cid, contract, contract_hash, topic0s, topic1s, topic2s, topic3s, log_contracts) = ($2, $3, $4, $5, $6, $7, $8, $9)`,
@@ -165,6 +175,19 @@ func (in *CIDIndexer) indexStateAndStorageCIDs(tx *sqlx.Tx, payload *CIDPayload,
return nil
}
+func (in *CIDIndexer) indexStateCID(tx *sqlx.Tx, stateNode StateNodeModel, headerID int64) (int64, error) {
+ var stateID int64
+ var stateKey string
+ if stateNode.StateKey != nullHash.String() {
+ stateKey = stateNode.StateKey
+ }
+ err := tx.QueryRowx(`INSERT INTO eth.state_cids (header_id, state_leaf_key, cid, state_path, node_type) VALUES ($1, $2, $3, $4, $5)
+ ON CONFLICT (header_id, state_path) DO UPDATE SET (state_leaf_key, cid, node_type) = ($2, $3, $5)
+ RETURNING id`,
+ headerID, stateKey, stateNode.CID, stateNode.Path, stateNode.NodeType).Scan(&stateID)
+ return stateID, err
+}
+
func (in *CIDIndexer) indexStateAccount(tx *sqlx.Tx, stateAccount StateAccountModel, stateID int64) error {
_, err := tx.Exec(`INSERT INTO eth.state_accounts (state_id, balance, nonce, code_hash, storage_root) VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (state_id) DO UPDATE SET (balance, nonce, code_hash, storage_root) = ($2, $3, $4, $5)`,
diff --git a/pkg/super_node/eth/ipld_fetcher.go b/pkg/super_node/eth/ipld_fetcher.go
index 64596b3a..ae3b5f8c 100644
--- a/pkg/super_node/eth/ipld_fetcher.go
+++ b/pkg/super_node/eth/ipld_fetcher.go
@@ -167,7 +167,6 @@ func (f *IPLDFetcher) FetchTrxs(cids []TxModel) ([]ipfs.BlockModel, error) {
// FetchRcts fetches receipts
// It uses the f.fetchBatch method
-// batch fetch preserves order?
func (f *IPLDFetcher) FetchRcts(cids []ReceiptModel) ([]ipfs.BlockModel, error) {
log.Debug("fetching receipt iplds")
rctCids := make([]cid.Cid, len(cids))
@@ -198,9 +197,9 @@ func (f *IPLDFetcher) FetchRcts(cids []ReceiptModel) ([]ipfs.BlockModel, error)
// needs to maintain the data's relation to state keys
func (f *IPLDFetcher) FetchState(cids []StateNodeModel) ([]StateNode, error) {
log.Debug("fetching state iplds")
- stateNodes := make([]StateNode, len(cids))
- for i, stateNode := range cids {
- if stateNode.CID == "" || stateNode.StateKey == "" {
+ stateNodes := make([]StateNode, 0, len(cids))
+ for _, stateNode := range cids {
+ if stateNode.CID == "" {
continue
}
dc, err := cid.Decode(stateNode.CID)
@@ -211,7 +210,7 @@ func (f *IPLDFetcher) FetchState(cids []StateNodeModel) ([]StateNode, error) {
if err != nil {
return nil, err
}
- stateNodes[i] = StateNode{
+ stateNodes = append(stateNodes, StateNode{
IPLD: ipfs.BlockModel{
Data: state.RawData(),
CID: state.Cid().String(),
@@ -219,7 +218,7 @@ func (f *IPLDFetcher) FetchState(cids []StateNodeModel) ([]StateNode, error) {
StateLeafKey: common.HexToHash(stateNode.StateKey),
Type: ResolveToNodeType(stateNode.NodeType),
Path: stateNode.Path,
- }
+ })
}
return stateNodes, nil
}
@@ -229,9 +228,9 @@ func (f *IPLDFetcher) FetchState(cids []StateNodeModel) ([]StateNode, error) {
// needs to maintain the data's relation to state and storage keys
func (f *IPLDFetcher) FetchStorage(cids []StorageNodeWithStateKeyModel) ([]StorageNode, error) {
log.Debug("fetching storage iplds")
- storageNodes := make([]StorageNode, len(cids))
- for i, storageNode := range cids {
- if storageNode.CID == "" || storageNode.StorageKey == "" || storageNode.StateKey == "" {
+ storageNodes := make([]StorageNode, 0, len(cids))
+ for _, storageNode := range cids {
+ if storageNode.CID == "" || storageNode.StateKey == "" {
continue
}
dc, err := cid.Decode(storageNode.CID)
@@ -242,7 +241,7 @@ func (f *IPLDFetcher) FetchStorage(cids []StorageNodeWithStateKeyModel) ([]Stora
if err != nil {
return nil, err
}
- storageNodes[i] = StorageNode{
+ storageNodes = append(storageNodes, StorageNode{
IPLD: ipfs.BlockModel{
Data: storage.RawData(),
CID: storage.Cid().String(),
@@ -251,7 +250,7 @@ func (f *IPLDFetcher) FetchStorage(cids []StorageNodeWithStateKeyModel) ([]Stora
StorageLeafKey: common.HexToHash(storageNode.StorageKey),
Type: ResolveToNodeType(storageNode.NodeType),
Path: storageNode.Path,
- }
+ })
}
return storageNodes, nil
}
diff --git a/pkg/super_node/eth/ipld_fetcher_test.go b/pkg/super_node/eth/ipld_fetcher_test.go
index 5192685d..d7bc9b73 100644
--- a/pkg/super_node/eth/ipld_fetcher_test.go
+++ b/pkg/super_node/eth/ipld_fetcher_test.go
@@ -89,8 +89,8 @@ var (
}
)
-var _ = Describe("Fetcher", func() {
- Describe("FetchCIDs", func() {
+var _ = Describe("IPLDFetcher", func() {
+ Describe("Fetch", func() {
BeforeEach(func() {
mockBlockService = new(mocks.MockIPFSBlockService)
err := mockBlockService.AddBlocks(mockBlocks)
diff --git a/pkg/super_node/eth/ipld_pg_fetcher.go b/pkg/super_node/eth/ipld_pg_fetcher.go
new file mode 100644
index 00000000..61fb53db
--- /dev/null
+++ b/pkg/super_node/eth/ipld_pg_fetcher.go
@@ -0,0 +1,215 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package eth
+
+import (
+ "errors"
+ "fmt"
+ "math/big"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/jmoiron/sqlx"
+ log "github.com/sirupsen/logrus"
+
+ "github.com/vulcanize/vulcanizedb/pkg/ipfs"
+ "github.com/vulcanize/vulcanizedb/pkg/postgres"
+ "github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+)
+
+// IPLDPGFetcher satisfies the IPLDFetcher interface for ethereum
+// It interfaces directly with PG-IPFS
+type IPLDPGFetcher struct {
+ db *postgres.DB
+}
+
+// NewIPLDPGFetcher creates a pointer to a new IPLDPGFetcher
+func NewIPLDPGFetcher(db *postgres.DB) *IPLDPGFetcher {
+ return &IPLDPGFetcher{
+ db: db,
+ }
+}
+
+// Fetch is the exported method for fetching and returning all the IPLDS specified in the CIDWrapper
+func (f *IPLDPGFetcher) Fetch(cids shared.CIDsForFetching) (shared.IPLDs, error) {
+ cidWrapper, ok := cids.(*CIDWrapper)
+ if !ok {
+ return nil, fmt.Errorf("eth fetcher: expected cids type %T got %T", &CIDWrapper{}, cids)
+ }
+ log.Debug("fetching iplds")
+ iplds := IPLDs{}
+ iplds.TotalDifficulty, ok = new(big.Int).SetString(cidWrapper.Header.TotalDifficulty, 10)
+ if !ok {
+ return nil, errors.New("eth fetcher: unable to set total difficulty")
+ }
+ iplds.BlockNumber = cidWrapper.BlockNumber
+
+ tx, err := f.db.Beginx()
+ if err != nil {
+ return nil, err
+ }
+ defer func() {
+ if p := recover(); p != nil {
+ shared.Rollback(tx)
+ panic(p)
+ } else if err != nil {
+ shared.Rollback(tx)
+ } else {
+ err = tx.Commit()
+ }
+ }()
+
+ iplds.Header, err = f.FetchHeader(tx, cidWrapper.Header)
+ if err != nil {
+ return nil, fmt.Errorf("eth pg fetcher: header fetching error: %s", err.Error())
+ }
+ iplds.Uncles, err = f.FetchUncles(tx, cidWrapper.Uncles)
+ if err != nil {
+ return nil, fmt.Errorf("eth pg fetcher: uncle fetching error: %s", err.Error())
+ }
+ iplds.Transactions, err = f.FetchTrxs(tx, cidWrapper.Transactions)
+ if err != nil {
+ return nil, fmt.Errorf("eth pg fetcher: transaction fetching error: %s", err.Error())
+ }
+ iplds.Receipts, err = f.FetchRcts(tx, cidWrapper.Receipts)
+ if err != nil {
+ return nil, fmt.Errorf("eth pg fetcher: receipt fetching error: %s", err.Error())
+ }
+ iplds.StateNodes, err = f.FetchState(tx, cidWrapper.StateNodes)
+ if err != nil {
+ return nil, fmt.Errorf("eth pg fetcher: state fetching error: %s", err.Error())
+ }
+ iplds.StorageNodes, err = f.FetchStorage(tx, cidWrapper.StorageNodes)
+ if err != nil {
+ return nil, fmt.Errorf("eth pg fetcher: storage fetching error: %s", err.Error())
+ }
+ return iplds, err
+}
+
+// FetchHeaders fetches headers
+func (f *IPLDPGFetcher) FetchHeader(tx *sqlx.Tx, c HeaderModel) (ipfs.BlockModel, error) {
+ log.Debug("fetching header ipld")
+ headerBytes, err := shared.FetchIPLD(tx, c.CID)
+ if err != nil {
+ return ipfs.BlockModel{}, err
+ }
+ return ipfs.BlockModel{
+ Data: headerBytes,
+ CID: c.CID,
+ }, nil
+}
+
+// FetchUncles fetches uncles
+func (f *IPLDPGFetcher) FetchUncles(tx *sqlx.Tx, cids []UncleModel) ([]ipfs.BlockModel, error) {
+ log.Debug("fetching uncle iplds")
+ uncleIPLDs := make([]ipfs.BlockModel, len(cids))
+ for i, c := range cids {
+ uncleBytes, err := shared.FetchIPLD(tx, c.CID)
+ if err != nil {
+ return nil, err
+ }
+ uncleIPLDs[i] = ipfs.BlockModel{
+ Data: uncleBytes,
+ CID: c.CID,
+ }
+ }
+ return uncleIPLDs, nil
+}
+
+// FetchTrxs fetches transactions
+func (f *IPLDPGFetcher) FetchTrxs(tx *sqlx.Tx, cids []TxModel) ([]ipfs.BlockModel, error) {
+ log.Debug("fetching transaction iplds")
+ trxIPLDs := make([]ipfs.BlockModel, len(cids))
+ for i, c := range cids {
+ txBytes, err := shared.FetchIPLD(tx, c.CID)
+ if err != nil {
+ return nil, err
+ }
+ trxIPLDs[i] = ipfs.BlockModel{
+ Data: txBytes,
+ CID: c.CID,
+ }
+ }
+ return trxIPLDs, nil
+}
+
+// FetchRcts fetches receipts
+func (f *IPLDPGFetcher) FetchRcts(tx *sqlx.Tx, cids []ReceiptModel) ([]ipfs.BlockModel, error) {
+ log.Debug("fetching receipt iplds")
+ rctIPLDs := make([]ipfs.BlockModel, len(cids))
+ for i, c := range cids {
+ rctBytes, err := shared.FetchIPLD(tx, c.CID)
+ if err != nil {
+ return nil, err
+ }
+ rctIPLDs[i] = ipfs.BlockModel{
+ Data: rctBytes,
+ CID: c.CID,
+ }
+ }
+ return rctIPLDs, nil
+}
+
+// FetchState fetches state nodes
+func (f *IPLDPGFetcher) FetchState(tx *sqlx.Tx, cids []StateNodeModel) ([]StateNode, error) {
+ log.Debug("fetching state iplds")
+ stateNodes := make([]StateNode, 0, len(cids))
+ for _, stateNode := range cids {
+ if stateNode.CID == "" {
+ continue
+ }
+ stateBytes, err := shared.FetchIPLD(tx, stateNode.CID)
+ if err != nil {
+ return nil, err
+ }
+ stateNodes = append(stateNodes, StateNode{
+ IPLD: ipfs.BlockModel{
+ Data: stateBytes,
+ CID: stateNode.CID,
+ },
+ StateLeafKey: common.HexToHash(stateNode.StateKey),
+ Type: ResolveToNodeType(stateNode.NodeType),
+ Path: stateNode.Path,
+ })
+ }
+ return stateNodes, nil
+}
+
+// FetchStorage fetches storage nodes
+func (f *IPLDPGFetcher) FetchStorage(tx *sqlx.Tx, cids []StorageNodeWithStateKeyModel) ([]StorageNode, error) {
+ log.Debug("fetching storage iplds")
+ storageNodes := make([]StorageNode, 0, len(cids))
+ for _, storageNode := range cids {
+ if storageNode.CID == "" || storageNode.StateKey == "" {
+ continue
+ }
+ storageBytes, err := shared.FetchIPLD(tx, storageNode.CID)
+ if err != nil {
+ return nil, err
+ }
+ storageNodes = append(storageNodes, StorageNode{
+ IPLD: ipfs.BlockModel{
+ Data: storageBytes,
+ CID: storageNode.CID,
+ },
+ StateLeafKey: common.HexToHash(storageNode.StateKey),
+ StorageLeafKey: common.HexToHash(storageNode.StorageKey),
+ Type: ResolveToNodeType(storageNode.NodeType),
+ Path: storageNode.Path,
+ })
+ }
+ return storageNodes, nil
+}
diff --git a/pkg/super_node/eth/ipld_pg_fetcher_test.go b/pkg/super_node/eth/ipld_pg_fetcher_test.go
new file mode 100644
index 00000000..d99643e2
--- /dev/null
+++ b/pkg/super_node/eth/ipld_pg_fetcher_test.go
@@ -0,0 +1,65 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package eth_test
+
+import (
+ . "github.com/onsi/ginkgo"
+ . "github.com/onsi/gomega"
+
+ "github.com/vulcanize/vulcanizedb/pkg/postgres"
+ "github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
+ "github.com/vulcanize/vulcanizedb/pkg/super_node/eth/mocks"
+ "github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+)
+
+var (
+ db *postgres.DB
+ pubAndIndexer *eth.IPLDPublisherAndIndexer
+ fetcher *eth.IPLDPGFetcher
+)
+
+var _ = Describe("IPLDPGFetcher", func() {
+ Describe("Fetch", func() {
+ BeforeEach(func() {
+ var err error
+ db, err = shared.SetupDB()
+ Expect(err).ToNot(HaveOccurred())
+ pubAndIndexer = eth.NewIPLDPublisherAndIndexer(db)
+ _, err = pubAndIndexer.Publish(mocks.MockConvertedPayload)
+ Expect(err).ToNot(HaveOccurred())
+ fetcher = eth.NewIPLDPGFetcher(db)
+ })
+ AfterEach(func() {
+ eth.TearDownDB(db)
+ })
+
+ It("Fetches and returns IPLDs for the CIDs provided in the CIDWrapper", func() {
+ i, err := fetcher.Fetch(mocks.MockCIDWrapper)
+ Expect(err).ToNot(HaveOccurred())
+ iplds, ok := i.(eth.IPLDs)
+ Expect(ok).To(BeTrue())
+ Expect(iplds.TotalDifficulty).To(Equal(mocks.MockConvertedPayload.TotalDifficulty))
+ Expect(iplds.BlockNumber).To(Equal(mocks.MockConvertedPayload.Block.Number()))
+ Expect(iplds.Header).To(Equal(mocks.MockIPLDs.Header))
+ Expect(len(iplds.Uncles)).To(Equal(0))
+ Expect(iplds.Transactions).To(Equal(mocks.MockIPLDs.Transactions))
+ Expect(iplds.Receipts).To(Equal(mocks.MockIPLDs.Receipts))
+ Expect(iplds.StateNodes).To(Equal(mocks.MockIPLDs.StateNodes))
+ Expect(iplds.StorageNodes).To(Equal(mocks.MockIPLDs.StorageNodes))
+ })
+ })
+})
diff --git a/pkg/super_node/eth/mocks/test_data.go b/pkg/super_node/eth/mocks/test_data.go
index aca169fc..af00a171 100644
--- a/pkg/super_node/eth/mocks/test_data.go
+++ b/pkg/super_node/eth/mocks/test_data.go
@@ -216,15 +216,15 @@ var (
})
nonce1 = uint64(1)
- contractRoot = "0x821e2556a290c86405f8160a2d662042a431ba456b9db265c79bb837c04be5f0"
- contractCodeHash = common.HexToHash("0x753f98a8d4328b15636e46f66f2cb4bc860100aa17967cc145fcd17d1d4710ea")
+ ContractRoot = "0x821e2556a290c86405f8160a2d662042a431ba456b9db265c79bb837c04be5f0"
+ ContractCodeHash = common.HexToHash("0x753f98a8d4328b15636e46f66f2cb4bc860100aa17967cc145fcd17d1d4710ea")
contractPathHash = crypto.Keccak256Hash([]byte{'\x06'})
ContractLeafKey = testhelpers.AddressToLeafKey(ContractAddress)
ContractAccount, _ = rlp.EncodeToBytes(state.Account{
Nonce: nonce1,
Balance: big.NewInt(0),
- CodeHash: contractCodeHash.Bytes(),
- Root: common.HexToHash(contractRoot),
+ CodeHash: ContractCodeHash.Bytes(),
+ Root: common.HexToHash(ContractRoot),
})
ContractPartialPath = common.Hex2Bytes("3114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45")
ContractLeafNode, _ = rlp.EncodeToBytes([]interface{}{
@@ -233,16 +233,16 @@ var (
})
nonce0 = uint64(0)
- accountRoot = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"
- accountCodeHash = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")
+ AccountRoot = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"
+ AccountCodeHash = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")
accountPathHash = crypto.Keccak256Hash([]byte{'\x0c'})
AccountAddresss = common.HexToAddress("0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e")
AccountLeafKey = testhelpers.Account2LeafKey
Account, _ = rlp.EncodeToBytes(state.Account{
Nonce: nonce0,
Balance: big.NewInt(1000),
- CodeHash: accountCodeHash.Bytes(),
- Root: common.HexToHash(accountRoot),
+ CodeHash: AccountCodeHash.Bytes(),
+ Root: common.HexToHash(AccountRoot),
})
AccountPartialPath = common.Hex2Bytes("3957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45")
AccountLeafNode, _ = rlp.EncodeToBytes([]interface{}{
@@ -374,14 +374,14 @@ var (
contractPathHash: {
Balance: big.NewInt(0).String(),
Nonce: nonce1,
- CodeHash: contractCodeHash.Bytes(),
- StorageRoot: common.HexToHash(contractRoot).String(),
+ CodeHash: ContractCodeHash.Bytes(),
+ StorageRoot: common.HexToHash(ContractRoot).String(),
},
accountPathHash: {
Balance: big.NewInt(1000).String(),
Nonce: nonce0,
- CodeHash: accountCodeHash.Bytes(),
- StorageRoot: common.HexToHash(accountRoot).String(),
+ CodeHash: AccountCodeHash.Bytes(),
+ StorageRoot: common.HexToHash(AccountRoot).String(),
},
},
}
diff --git a/pkg/super_node/eth/publishAndIndexer.go b/pkg/super_node/eth/publishAndIndexer.go
new file mode 100644
index 00000000..f12109f8
--- /dev/null
+++ b/pkg/super_node/eth/publishAndIndexer.go
@@ -0,0 +1,227 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package eth
+
+import (
+ "fmt"
+
+ "github.com/ethereum/go-ethereum/core/state"
+ "github.com/ethereum/go-ethereum/crypto"
+ "github.com/ethereum/go-ethereum/rlp"
+ "github.com/ethereum/go-ethereum/statediff"
+ "github.com/jmoiron/sqlx"
+
+ common2 "github.com/vulcanize/vulcanizedb/pkg/eth/converters/common"
+ "github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld"
+ "github.com/vulcanize/vulcanizedb/pkg/postgres"
+ "github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+)
+
+// IPLDPublisherAndIndexer satisfies the IPLDPublisher interface for ethereum
+// It interfaces directly with the public.blocks table of PG-IPFS rather than going through an ipfs intermediary
+// It publishes and indexes IPLDs together in a single sqlx.Tx
+type IPLDPublisherAndIndexer struct {
+ indexer *CIDIndexer
+}
+
+// NewIPLDPublisherAndIndexer creates a pointer to a new IPLDPublisherAndIndexer which satisfies the IPLDPublisher interface
+func NewIPLDPublisherAndIndexer(db *postgres.DB) *IPLDPublisherAndIndexer {
+ return &IPLDPublisherAndIndexer{
+ indexer: NewCIDIndexer(db),
+ }
+}
+
+// Publish publishes an IPLDPayload to IPFS and returns the corresponding CIDPayload
+func (pub *IPLDPublisherAndIndexer) Publish(payload shared.ConvertedData) (shared.CIDsForIndexing, error) {
+ ipldPayload, ok := payload.(ConvertedPayload)
+ if !ok {
+ return nil, fmt.Errorf("eth publisher expected payload type %T got %T", ConvertedPayload{}, payload)
+ }
+ // Generate the iplds
+ headerNode, uncleNodes, txNodes, txTrieNodes, rctNodes, rctTrieNodes, err := ipld.FromBlockAndReceipts(ipldPayload.Block, ipldPayload.Receipts)
+ if err != nil {
+ return nil, err
+ }
+
+ // Begin new db tx
+ tx, err := pub.indexer.db.Beginx()
+ if err != nil {
+ return nil, err
+ }
+ defer func() {
+ if p := recover(); p != nil {
+ shared.Rollback(tx)
+ panic(p)
+ } else if err != nil {
+ shared.Rollback(tx)
+ } else {
+ err = tx.Commit()
+ }
+ }()
+
+ // Publish trie nodes
+ for _, node := range txTrieNodes {
+ if err := shared.PublishIPLD(tx, node); err != nil {
+ return nil, err
+ }
+ }
+ for _, node := range rctTrieNodes {
+ if err := shared.PublishIPLD(tx, node); err != nil {
+ return nil, err
+ }
+ }
+
+ // Publish and index header
+ if err := shared.PublishIPLD(tx, headerNode); err != nil {
+ return nil, err
+ }
+ reward := common2.CalcEthBlockReward(ipldPayload.Block.Header(), ipldPayload.Block.Uncles(), ipldPayload.Block.Transactions(), ipldPayload.Receipts)
+ header := HeaderModel{
+ CID: headerNode.Cid().String(),
+ ParentHash: ipldPayload.Block.ParentHash().String(),
+ BlockNumber: ipldPayload.Block.Number().String(),
+ BlockHash: ipldPayload.Block.Hash().String(),
+ TotalDifficulty: ipldPayload.TotalDifficulty.String(),
+ Reward: reward.String(),
+ Bloom: ipldPayload.Block.Bloom().Bytes(),
+ StateRoot: ipldPayload.Block.Root().String(),
+ RctRoot: ipldPayload.Block.ReceiptHash().String(),
+ TxRoot: ipldPayload.Block.TxHash().String(),
+ UncleRoot: ipldPayload.Block.UncleHash().String(),
+ Timestamp: ipldPayload.Block.Time(),
+ }
+ headerID, err := pub.indexer.indexHeaderCID(tx, header)
+ if err != nil {
+ return nil, err
+ }
+
+ // Publish and index uncles
+ for _, uncleNode := range uncleNodes {
+ if err := shared.PublishIPLD(tx, uncleNode); err != nil {
+ return nil, err
+ }
+ uncleReward := common2.CalcUncleMinerReward(ipldPayload.Block.Number().Int64(), uncleNode.Number.Int64())
+ uncle := UncleModel{
+ CID: uncleNode.Cid().String(),
+ ParentHash: uncleNode.ParentHash.String(),
+ BlockHash: uncleNode.Hash().String(),
+ Reward: uncleReward.String(),
+ }
+ if err := pub.indexer.indexUncleCID(tx, uncle, headerID); err != nil {
+ return nil, err
+ }
+ }
+
+ // Publish and index txs and receipts
+ for i, txNode := range txNodes {
+ if err := shared.PublishIPLD(tx, txNode); err != nil {
+ return nil, err
+ }
+ rctNode := rctNodes[i]
+ if err := shared.PublishIPLD(tx, rctNode); err != nil {
+ return nil, err
+ }
+ txModel := ipldPayload.TxMetaData[i]
+ txModel.CID = txNode.Cid().String()
+ txID, err := pub.indexer.indexTransactionCID(tx, txModel, headerID)
+ if err != nil {
+ return nil, err
+ }
+ rctModel := ipldPayload.ReceiptMetaData[i]
+ rctModel.CID = rctNode.Cid().String()
+ if err := pub.indexer.indexReceiptCID(tx, rctModel, txID); err != nil {
+ return nil, err
+ }
+ }
+
+ // Publish and index state and storage
+ err = pub.publishAndIndexStateAndStorage(tx, ipldPayload, headerID)
+
+ // This IPLDPublisher does both publishing and indexing, we do not need to pass anything forward to the indexer
+ return nil, err // return err variable explicitly so that we return the err = tx.Commit() assignment in the defer
+}
+
+func (pub *IPLDPublisherAndIndexer) publishAndIndexStateAndStorage(tx *sqlx.Tx, ipldPayload ConvertedPayload, headerID int64) error {
+ // Publish and index state and storage
+ for _, stateNode := range ipldPayload.StateNodes {
+ stateIPLD, err := ipld.FromStateTrieRLP(stateNode.Value)
+ if err != nil {
+ return err
+ }
+ if err := shared.PublishIPLD(tx, stateIPLD); err != nil {
+ return err
+ }
+ stateModel := StateNodeModel{
+ Path: stateNode.Path,
+ StateKey: stateNode.LeafKey.String(),
+ CID: stateIPLD.Cid().String(),
+ NodeType: ResolveFromNodeType(stateNode.Type),
+ }
+ stateID, err := pub.indexer.indexStateCID(tx, stateModel, headerID)
+ if err != nil {
+ return err
+ }
+ // If we have a leaf, decode and index the account data and publish and index any associated storage diffs
+ if stateNode.Type == statediff.Leaf {
+ var i []interface{}
+ if err := rlp.DecodeBytes(stateNode.Value, &i); err != nil {
+ return err
+ }
+ if len(i) != 2 {
+ return fmt.Errorf("IPLDPublisherAndIndexer expected state leaf node rlp to decode into two elements")
+ }
+ var account state.Account
+ if err := rlp.DecodeBytes(i[1].([]byte), &account); err != nil {
+ return err
+ }
+ accountModel := StateAccountModel{
+ Balance: account.Balance.String(),
+ Nonce: account.Nonce,
+ CodeHash: account.CodeHash,
+ StorageRoot: account.Root.String(),
+ }
+ if err := pub.indexer.indexStateAccount(tx, accountModel, stateID); err != nil {
+ return err
+ }
+ statePathHash := crypto.Keccak256Hash(stateNode.Path)
+ for _, storageNode := range ipldPayload.StorageNodes[statePathHash] {
+ storageIPLD, err := ipld.FromStorageTrieRLP(storageNode.Value)
+ if err != nil {
+ return err
+ }
+ if err := shared.PublishIPLD(tx, storageIPLD); err != nil {
+ return err
+ }
+ storageModel := StorageNodeModel{
+ Path: storageNode.Path,
+ StorageKey: storageNode.LeafKey.Hex(),
+ CID: storageIPLD.Cid().String(),
+ NodeType: ResolveFromNodeType(storageNode.Type),
+ }
+ if err := pub.indexer.indexStorageCID(tx, storageModel, stateID); err != nil {
+ return err
+ }
+ }
+ }
+ }
+ return nil
+}
+
+// Index satisfies the shared.CIDIndexer interface
+func (pub *IPLDPublisherAndIndexer) Index(cids shared.CIDsForIndexing) error {
+ return nil
+}
diff --git a/pkg/super_node/eth/publishAndIndexer_test.go b/pkg/super_node/eth/publishAndIndexer_test.go
new file mode 100644
index 00000000..775b2bb6
--- /dev/null
+++ b/pkg/super_node/eth/publishAndIndexer_test.go
@@ -0,0 +1,237 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package eth_test
+
+import (
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ipfs/go-cid"
+ "github.com/ipfs/go-ipfs-blockstore"
+ "github.com/ipfs/go-ipfs-ds-help"
+ . "github.com/onsi/ginkgo"
+ . "github.com/onsi/gomega"
+ "github.com/vulcanize/vulcanizedb/pkg/postgres"
+ "github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
+ "github.com/vulcanize/vulcanizedb/pkg/super_node/eth/mocks"
+ "github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+)
+
+var _ = Describe("PublishAndIndexer", func() {
+ var (
+ db *postgres.DB
+ err error
+ repo *eth.IPLDPublisherAndIndexer
+ ipfsPgGet = `SELECT data FROM public.blocks
+ WHERE key = $1`
+ )
+ BeforeEach(func() {
+ db, err = shared.SetupDB()
+ Expect(err).ToNot(HaveOccurred())
+ repo = eth.NewIPLDPublisherAndIndexer(db)
+ })
+ AfterEach(func() {
+ eth.TearDownDB(db)
+ })
+
+ Describe("Publish", func() {
+ It("Published and indexes header IPLDs in a single tx", func() {
+ emptyReturn, err := repo.Publish(mocks.MockConvertedPayload)
+ Expect(emptyReturn).To(BeNil())
+ Expect(err).ToNot(HaveOccurred())
+ pgStr := `SELECT cid, td, reward, id
+ FROM eth.header_cids
+ WHERE block_number = $1`
+ // check header was properly indexed
+ type res struct {
+ CID string
+ TD string
+ Reward string
+ ID int
+ }
+ header := new(res)
+ err = db.QueryRowx(pgStr, 1).StructScan(header)
+ Expect(err).ToNot(HaveOccurred())
+ Expect(header.CID).To(Equal(mocks.HeaderCID.String()))
+ Expect(header.TD).To(Equal(mocks.MockBlock.Difficulty().String()))
+ Expect(header.Reward).To(Equal("5000000000000000000"))
+ dc, err := cid.Decode(header.CID)
+ Expect(err).ToNot(HaveOccurred())
+ mhKey := dshelp.CidToDsKey(dc)
+ prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
+ var data []byte
+ err = db.Get(&data, ipfsPgGet, prefixedKey)
+ Expect(err).ToNot(HaveOccurred())
+ Expect(data).To(Equal(mocks.MockHeaderRlp))
+ })
+
+ It("Publishes and indexes transaction IPLDs in a single tx", func() {
+ emptyReturn, err := repo.Publish(mocks.MockConvertedPayload)
+ Expect(emptyReturn).To(BeNil())
+ Expect(err).ToNot(HaveOccurred())
+ // check that txs were properly indexed
+ trxs := make([]string, 0)
+ pgStr := `SELECT transaction_cids.cid FROM eth.transaction_cids INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id)
+ WHERE header_cids.block_number = $1`
+ err = db.Select(&trxs, pgStr, 1)
+ Expect(err).ToNot(HaveOccurred())
+ Expect(len(trxs)).To(Equal(3))
+ Expect(shared.ListContainsString(trxs, mocks.Trx1CID.String())).To(BeTrue())
+ Expect(shared.ListContainsString(trxs, mocks.Trx2CID.String())).To(BeTrue())
+ Expect(shared.ListContainsString(trxs, mocks.Trx3CID.String())).To(BeTrue())
+ // and published
+ for _, c := range trxs {
+ dc, err := cid.Decode(c)
+ Expect(err).ToNot(HaveOccurred())
+ mhKey := dshelp.CidToDsKey(dc)
+ prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
+ var data []byte
+ err = db.Get(&data, ipfsPgGet, prefixedKey)
+ Expect(err).ToNot(HaveOccurred())
+ switch c {
+ case mocks.Trx1CID.String():
+ Expect(data).To(Equal(mocks.MockTransactions.GetRlp(0)))
+ case mocks.Trx2CID.String():
+ Expect(data).To(Equal(mocks.MockTransactions.GetRlp(1)))
+ case mocks.Trx3CID.String():
+ Expect(data).To(Equal(mocks.MockTransactions.GetRlp(2)))
+ }
+ }
+ })
+
+ It("Publishes and indexes receipt IPLDs in a single tx", func() {
+ emptyReturn, err := repo.Publish(mocks.MockConvertedPayload)
+ Expect(emptyReturn).To(BeNil())
+ Expect(err).ToNot(HaveOccurred())
+ // check receipts were properly indexed
+ rcts := make([]string, 0)
+ pgStr := `SELECT receipt_cids.cid FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
+ WHERE receipt_cids.tx_id = transaction_cids.id
+ AND transaction_cids.header_id = header_cids.id
+ AND header_cids.block_number = $1`
+ err = db.Select(&rcts, pgStr, 1)
+ Expect(err).ToNot(HaveOccurred())
+ Expect(len(rcts)).To(Equal(3))
+ Expect(shared.ListContainsString(rcts, mocks.Rct1CID.String())).To(BeTrue())
+ Expect(shared.ListContainsString(rcts, mocks.Rct2CID.String())).To(BeTrue())
+ Expect(shared.ListContainsString(rcts, mocks.Rct3CID.String())).To(BeTrue())
+ // and published
+ for _, c := range rcts {
+ dc, err := cid.Decode(c)
+ Expect(err).ToNot(HaveOccurred())
+ mhKey := dshelp.CidToDsKey(dc)
+ prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
+ var data []byte
+ err = db.Get(&data, ipfsPgGet, prefixedKey)
+ Expect(err).ToNot(HaveOccurred())
+ switch c {
+ case mocks.Rct1CID.String():
+ Expect(data).To(Equal(mocks.MockReceipts.GetRlp(0)))
+ case mocks.Rct2CID.String():
+ Expect(data).To(Equal(mocks.MockReceipts.GetRlp(1)))
+ case mocks.Rct3CID.String():
+ Expect(data).To(Equal(mocks.MockReceipts.GetRlp(2)))
+ }
+ }
+ })
+
+ It("Publishes and indexes state IPLDs in a single tx", func() {
+ emptyReturn, err := repo.Publish(mocks.MockConvertedPayload)
+ Expect(emptyReturn).To(BeNil())
+ Expect(err).ToNot(HaveOccurred())
+ // check that state nodes were properly indexed and published
+ stateNodes := make([]eth.StateNodeModel, 0)
+ pgStr := `SELECT state_cids.id, state_cids.cid, state_cids.state_leaf_key, state_cids.node_type, state_cids.state_path, state_cids.header_id
+ FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
+ WHERE header_cids.block_number = $1`
+ err = db.Select(&stateNodes, pgStr, 1)
+ Expect(err).ToNot(HaveOccurred())
+ Expect(len(stateNodes)).To(Equal(2))
+ for _, stateNode := range stateNodes {
+ var data []byte
+ dc, err := cid.Decode(stateNode.CID)
+ Expect(err).ToNot(HaveOccurred())
+ mhKey := dshelp.CidToDsKey(dc)
+ prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
+ err = db.Get(&data, ipfsPgGet, prefixedKey)
+ Expect(err).ToNot(HaveOccurred())
+ pgStr = `SELECT * from eth.state_accounts WHERE state_id = $1`
+ var account eth.StateAccountModel
+ err = db.Get(&account, pgStr, stateNode.ID)
+ Expect(err).ToNot(HaveOccurred())
+ if stateNode.CID == mocks.State1CID.String() {
+ Expect(stateNode.NodeType).To(Equal(2))
+ Expect(stateNode.StateKey).To(Equal(common.BytesToHash(mocks.ContractLeafKey).Hex()))
+ Expect(stateNode.Path).To(Equal([]byte{'\x06'}))
+ Expect(data).To(Equal(mocks.ContractLeafNode))
+ Expect(account).To(Equal(eth.StateAccountModel{
+ ID: account.ID,
+ StateID: stateNode.ID,
+ Balance: "0",
+ CodeHash: mocks.ContractCodeHash.Bytes(),
+ StorageRoot: mocks.ContractRoot,
+ Nonce: 1,
+ }))
+ }
+ if stateNode.CID == mocks.State2CID.String() {
+ Expect(stateNode.NodeType).To(Equal(2))
+ Expect(stateNode.StateKey).To(Equal(common.BytesToHash(mocks.AccountLeafKey).Hex()))
+ Expect(stateNode.Path).To(Equal([]byte{'\x0c'}))
+ Expect(data).To(Equal(mocks.AccountLeafNode))
+ Expect(account).To(Equal(eth.StateAccountModel{
+ ID: account.ID,
+ StateID: stateNode.ID,
+ Balance: "1000",
+ CodeHash: mocks.AccountCodeHash.Bytes(),
+ StorageRoot: mocks.AccountRoot,
+ Nonce: 0,
+ }))
+ }
+ }
+ pgStr = `SELECT * from eth.state_accounts WHERE state_id = $1`
+ })
+
+ It("Publishes and indexes storage IPLDs in a single tx", func() {
+ emptyReturn, err := repo.Publish(mocks.MockConvertedPayload)
+ Expect(emptyReturn).To(BeNil())
+ Expect(err).ToNot(HaveOccurred())
+ // check that storage nodes were properly indexed
+ storageNodes := make([]eth.StorageNodeWithStateKeyModel, 0)
+ pgStr := `SELECT storage_cids.cid, state_cids.state_leaf_key, storage_cids.storage_leaf_key, storage_cids.node_type, storage_cids.storage_path
+ FROM eth.storage_cids, eth.state_cids, eth.header_cids
+ WHERE storage_cids.state_id = state_cids.id
+ AND state_cids.header_id = header_cids.id
+ AND header_cids.block_number = $1`
+ err = db.Select(&storageNodes, pgStr, 1)
+ Expect(err).ToNot(HaveOccurred())
+ Expect(len(storageNodes)).To(Equal(1))
+ Expect(storageNodes[0]).To(Equal(eth.StorageNodeWithStateKeyModel{
+ CID: mocks.StorageCID.String(),
+ NodeType: 2,
+ StorageKey: common.BytesToHash(mocks.StorageLeafKey).Hex(),
+ StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(),
+ Path: []byte{},
+ }))
+ var data []byte
+ dc, err := cid.Decode(storageNodes[0].CID)
+ Expect(err).ToNot(HaveOccurred())
+ mhKey := dshelp.CidToDsKey(dc)
+ prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
+ err = db.Get(&data, ipfsPgGet, prefixedKey)
+ Expect(err).ToNot(HaveOccurred())
+ Expect(data).To(Equal(mocks.StorageLeafNode))
+ })
+ })
+})
diff --git a/pkg/super_node/eth/publisher.go b/pkg/super_node/eth/publisher.go
index 07a323ed..7b8a0cd4 100644
--- a/pkg/super_node/eth/publisher.go
+++ b/pkg/super_node/eth/publisher.go
@@ -45,7 +45,7 @@ type IPLDPublisher struct {
StoragePutter ipfs.DagPutter
}
-// NewIPLDPublisher creates a pointer to a new Publisher which satisfies the IPLDPublisher interface
+// NewIPLDPublisher creates a pointer to a new IPLDPublisher which satisfies the IPLDPublisher interface
func NewIPLDPublisher(ipfsPath string) (*IPLDPublisher, error) {
node, err := ipfs.InitIPFSNode(ipfsPath)
if err != nil {
diff --git a/pkg/super_node/resync/config.go b/pkg/super_node/resync/config.go
index bd35c00c..e43460a6 100644
--- a/pkg/super_node/resync/config.go
+++ b/pkg/super_node/resync/config.go
@@ -52,6 +52,7 @@ type Config struct {
DB *postgres.DB
DBConfig config.Database
IPFSPath string
+ IPFSMode shared.IPFSMode
HTTPClient interface{} // Note this client is expected to support the retrieval of the specified data type(s)
NodeInfo core.Node // Info for the associated node
@@ -81,8 +82,8 @@ func NewReSyncConfig() (*Config, error) {
viper.BindEnv("resync.timeout", shared.HTTP_TIMEOUT)
timeout := viper.GetInt("resync.timeout")
- if timeout < 15 {
- timeout = 15
+ if timeout < 5 {
+ timeout = 5
}
c.Timeout = time.Second * time.Duration(timeout)
@@ -92,12 +93,18 @@ func NewReSyncConfig() (*Config, error) {
c.ClearOldCache = viper.GetBool("resync.clearOldCache")
c.ResetValidation = viper.GetBool("resync.resetValidation")
- c.IPFSPath, err = shared.GetIPFSPath()
+ c.IPFSMode, err = shared.GetIPFSMode()
if err != nil {
return nil, err
}
+ if c.IPFSMode == shared.LocalInterface || c.IPFSMode == shared.RemoteClient {
+ c.IPFSPath, err = shared.GetIPFSPath()
+ if err != nil {
+ return nil, err
+ }
+ }
resyncType := viper.GetString("resync.type")
- c.ResyncType, err = shared.GenerateResyncTypeFromString(resyncType)
+ c.ResyncType, err = shared.GenerateDataTypeFromString(resyncType)
if err != nil {
return nil, err
}
@@ -106,7 +113,7 @@ func NewReSyncConfig() (*Config, error) {
if err != nil {
return nil, err
}
- if ok, err := shared.SupportedResyncType(c.ResyncType, c.Chain); !ok {
+ if ok, err := shared.SupportedDataType(c.ResyncType, c.Chain); !ok {
if err != nil {
return nil, err
}
diff --git a/pkg/super_node/resync/service.go b/pkg/super_node/resync/service.go
index 1e291ff1..b5c43f0f 100644
--- a/pkg/super_node/resync/service.go
+++ b/pkg/super_node/resync/service.go
@@ -22,7 +22,7 @@ import (
"github.com/sirupsen/logrus"
- "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+ utils "github.com/vulcanize/vulcanizedb/libraries/shared/utilities"
"github.com/vulcanize/vulcanizedb/pkg/super_node"
"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
)
@@ -64,11 +64,11 @@ type Service struct {
// NewResyncService creates and returns a resync service from the provided settings
func NewResyncService(settings *Config) (Resync, error) {
- publisher, err := super_node.NewIPLDPublisher(settings.Chain, settings.IPFSPath)
+ publisher, err := super_node.NewIPLDPublisher(settings.Chain, settings.IPFSPath, settings.DB, settings.IPFSMode)
if err != nil {
return nil, err
}
- indexer, err := super_node.NewCIDIndexer(settings.Chain, settings.DB)
+ indexer, err := super_node.NewCIDIndexer(settings.Chain, settings.DB, settings.IPFSMode)
if err != nil {
return nil, err
}
diff --git a/pkg/super_node/service.go b/pkg/super_node/service.go
index 53b5bb24..c952a2db 100644
--- a/pkg/super_node/service.go
+++ b/pkg/super_node/service.go
@@ -34,7 +34,7 @@ import (
)
const (
- PayloadChanBufferSize = 20000
+ PayloadChanBufferSize = 2000
)
// SuperNode is the top level interface for streaming, converting to IPLDs, publishing,
@@ -109,11 +109,11 @@ func NewSuperNode(settings *Config) (SuperNode, error) {
if err != nil {
return nil, err
}
- sn.Publisher, err = NewIPLDPublisher(settings.Chain, settings.IPFSPath)
+ sn.Publisher, err = NewIPLDPublisher(settings.Chain, settings.IPFSPath, settings.DB, settings.IPFSMode)
if err != nil {
return nil, err
}
- sn.Indexer, err = NewCIDIndexer(settings.Chain, settings.DB)
+ sn.Indexer, err = NewCIDIndexer(settings.Chain, settings.DB, settings.IPFSMode)
if err != nil {
return nil, err
}
@@ -128,7 +128,7 @@ func NewSuperNode(settings *Config) (SuperNode, error) {
if err != nil {
return nil, err
}
- sn.IPLDFetcher, err = NewIPLDFetcher(settings.Chain, settings.IPFSPath)
+ sn.IPLDFetcher, err = NewIPLDFetcher(settings.Chain, settings.IPFSPath, settings.DB, settings.IPFSMode)
if err != nil {
return nil, err
}
@@ -220,7 +220,13 @@ func (sap *Service) Sync(wg *sync.WaitGroup, screenAndServePayload chan<- shared
default:
}
// Forward the payload to the publishAndIndex workers
- publishAndIndexPayload <- ipldPayload
+ // this channel acts as a ring buffer
+ select {
+ case publishAndIndexPayload <- ipldPayload:
+ default:
+ <-publishAndIndexPayload
+ publishAndIndexPayload <- ipldPayload
+ }
case err := <-sub.Err():
log.Errorf("super node subscription error for chain %s: %v", sap.chain.String(), err)
case <-sap.QuitChan:
@@ -244,12 +250,12 @@ func (sap *Service) publishAndIndex(id int, publishAndIndexPayload <-chan shared
log.Debugf("publishing %s data streamed at head height %d", sap.chain.String(), payload.Height())
cidPayload, err := sap.Publisher.Publish(payload)
if err != nil {
- log.Errorf("super node publishAndIndex worker %d error for chain %s: %v", id, sap.chain.String(), err)
+ log.Errorf("super node publishAndIndex worker %d publishing error for chain %s: %v", id, sap.chain.String(), err)
continue
}
log.Debugf("indexing %s data streamed at head height %d", sap.chain.String(), payload.Height())
if err := sap.Indexer.Index(cidPayload); err != nil {
- log.Errorf("super node publishAndIndex worker %d error for chain %s: %v", id, sap.chain.String(), err)
+ log.Errorf("super node publishAndIndex worker %d indexing error for chain %s: %v", id, sap.chain.String(), err)
}
}
}
diff --git a/pkg/super_node/shared/data_type.go b/pkg/super_node/shared/data_type.go
index 8112988a..d62694df 100644
--- a/pkg/super_node/shared/data_type.go
+++ b/pkg/super_node/shared/data_type.go
@@ -57,8 +57,8 @@ func (r DataType) String() string {
}
}
-// GenerateResyncTypeFromString
-func GenerateResyncTypeFromString(str string) (DataType, error) {
+// GenerateDataTypeFromString
+func GenerateDataTypeFromString(str string) (DataType, error) {
switch strings.ToLower(str) {
case "full", "f":
return Full, nil
@@ -79,7 +79,7 @@ func GenerateResyncTypeFromString(str string) (DataType, error) {
}
}
-func SupportedResyncType(d DataType, c ChainType) (bool, error) {
+func SupportedDataType(d DataType, c ChainType) (bool, error) {
switch c {
case Ethereum:
switch d {
diff --git a/pkg/super_node/shared/env.go b/pkg/super_node/shared/env.go
index 4a895f29..b49a54c3 100644
--- a/pkg/super_node/shared/env.go
+++ b/pkg/super_node/shared/env.go
@@ -30,6 +30,7 @@ import (
// Env variables
const (
IPFS_PATH = "IPFS_PATH"
+ IPFS_MODE = "IPFS_MODE"
HTTP_TIMEOUT = "HTTP_TIMEOUT"
ETH_WS_PATH = "ETH_WS_PATH"
@@ -82,6 +83,16 @@ func GetIPFSPath() (string, error) {
return ipfsPath, nil
}
+// GetIPFSMode returns the ipfs mode of operation from the config or env variable
+func GetIPFSMode() (IPFSMode, error) {
+ viper.BindEnv("ipfs.mode", IPFS_MODE)
+ ipfsMode := viper.GetString("ipfs.mode")
+ if ipfsMode == "" {
+ return DirectPostgres, nil
+ }
+ return NewIPFSMode(ipfsMode)
+}
+
// GetBtcNodeAndClient returns btc node info from path url
func GetBtcNodeAndClient(path string) (core.Node, *rpcclient.ConnConfig) {
viper.BindEnv("bitcoin.nodeID", BTC_NODE_ID)
diff --git a/pkg/super_node/shared/functions.go b/pkg/super_node/shared/functions.go
index ef338fb7..ca85202d 100644
--- a/pkg/super_node/shared/functions.go
+++ b/pkg/super_node/shared/functions.go
@@ -19,7 +19,14 @@ package shared
import (
"bytes"
+ "github.com/ipfs/go-cid"
+
"github.com/ethereum/go-ethereum/common"
+ "github.com/ipfs/go-ipfs-blockstore"
+ "github.com/ipfs/go-ipfs-ds-help"
+ node "github.com/ipfs/go-ipld-format"
+ "github.com/jmoiron/sqlx"
+ "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
)
@@ -69,3 +76,40 @@ func HandleNullAddr(to common.Address) string {
}
return to.Hex()
}
+
+// Rollback sql transaction and log any error
+func Rollback(tx *sqlx.Tx) {
+ if err := tx.Rollback(); err != nil {
+ logrus.Error(err)
+ }
+}
+
+// PublishIPLD is used to insert an ipld into Postgres blockstore with the provided tx
+func PublishIPLD(tx *sqlx.Tx, i node.Node) error {
+ dbKey := dshelp.CidToDsKey(i.Cid())
+ prefixedKey := blockstore.BlockPrefix.String() + dbKey.String()
+ raw := i.RawData()
+ _, err := tx.Exec(`INSERT INTO public.blocks (key, data) VALUES ($1, $2) ON CONFLICT (key) DO NOTHING`, prefixedKey, raw)
+ return err
+}
+
+// FetchIPLD is used to retrieve an ipld from Postgres blockstore with the provided tx
+func FetchIPLD(tx *sqlx.Tx, cid string) ([]byte, error) {
+ mhKey, err := MultihashKeyFromCIDString(cid)
+ if err != nil {
+ return nil, err
+ }
+ pgStr := `SELECT data FROM public.blocks WHERE key = $1`
+ var block []byte
+ return block, tx.Get(&block, pgStr, mhKey)
+}
+
+// MultihashKeyFromCIDString converts a cid string into a blockstore-prefixed multihash db key string
+func MultihashKeyFromCIDString(c string) (string, error) {
+ dc, err := cid.Decode(c)
+ if err != nil {
+ return "", err
+ }
+ dbKey := dshelp.CidToDsKey(dc)
+ return blockstore.BlockPrefix.String() + dbKey.String(), nil
+}
diff --git a/pkg/super_node/shared/ipfs_mode.go b/pkg/super_node/shared/ipfs_mode.go
new file mode 100644
index 00000000..e6eb7f25
--- /dev/null
+++ b/pkg/super_node/shared/ipfs_mode.go
@@ -0,0 +1,58 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package shared
+
+import (
+ "errors"
+ "strings"
+)
+
+// IPFSMode enum for specifying how we want to interface and publish objects to IPFS
+type IPFSMode int
+
+const (
+ Unknown IPFSMode = iota
+ LocalInterface
+ RemoteClient
+ DirectPostgres
+)
+
+func (c IPFSMode) String() string {
+ switch c {
+ case LocalInterface:
+ return "Local"
+ case RemoteClient:
+ return "Remote"
+ case DirectPostgres:
+ return "Postgres"
+ default:
+ return ""
+ }
+}
+
+func NewIPFSMode(name string) (IPFSMode, error) {
+ switch strings.ToLower(name) {
+ case "local", "interface", "minimal":
+ return LocalInterface, nil
+ case "remote", "client":
+ return RemoteClient, errors.New("remote IPFS client mode is not currently supported")
+ case "postgres", "direct":
+ return DirectPostgres, nil
+ default:
+ return Unknown, errors.New("unrecognized name for ipfs mode")
+ }
+}
diff --git a/version/version.go b/version/version.go
index 43c8f6ed..b36e5f4a 100644
--- a/version/version.go
+++ b/version/version.go
@@ -19,22 +19,22 @@ package version
import "fmt"
const (
- VersionMajor = 0 // Major version component of the current release
- VersionMinor = 1 // Minor version component of the current release
- VersionPatch = 1 // Patch version component of the current release
- VersionMeta = "alpha" // Version metadata to append to the version string
+ Major = 0 // Major version component of the current release
+ Minor = 1 // Minor version component of the current release
+ Patch = 2 // Patch version component of the current release
+ Meta = "alpha" // Version metadata to append to the version string
)
// Version holds the textual version string.
var Version = func() string {
- return fmt.Sprintf("%d.%d.%d", VersionMajor, VersionMinor, VersionPatch)
+ return fmt.Sprintf("%d.%d.%d", Major, Minor, Patch)
}()
// VersionWithMeta holds the textual version string including the metadata.
var VersionWithMeta = func() string {
v := Version
- if VersionMeta != "" {
- v += "-" + VersionMeta
+ if Meta != "" {
+ v += "-" + Meta
}
return v
}()