cerc-io/go-ethereum
18
0
Fork 0
Code Issues 9 Pull Requests Packages Projects Releases 66 Wiki Activity

README updates #186

Closed
justindotpub wants to merge 9 commits from jj-docs into statediff
pull from: jj-docs
merge into: cerc-io:statediff
Conversation 1 Commits 9 Files Changed 140 +15698 -274
140 changed files with 15698 additions and 274 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
.github/workflows/checks.yml vendored Normal file
View File

@ -0,0 +1,14 @@
name: checks
on: [pull_request]
jobs:
linter-check:
runs-on: ubuntu-latest
steps:
- uses: actions/setup-go@v2
with:
go-version: '1.16.x'
- uses: actions/checkout@v2
- name: Run linter
run: go run build/ci.go lint

40
.github/workflows/on-master.yaml vendored Normal file
View File

@ -0,0 +1,40 @@
name: Docker Build and publish to Github
on:
push:
branches:
- v1.10.13-statediff
- v1.10.12-statediff
- v1.10.11-statediff
- v1.10.10-statediff
- v1.10.9-statediff
- v1.10.8-statediff
- v1.10.7-statediff
- v1.10.6-statediff
- v1.10.5-statediff
- v1.10.4-statediff
- v1.10.3-statediff
- v1.10.2-statediff
- v1.10.1-statediff
- v1.9.25-statediff
- v1.9.24-statediff
- v1.9.23-statediff
- v1.9.11-statediff
jobs:
build:
name: Run docker build and publish
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Run docker build
run: docker build -t vulcanize/go-ethereum -f Dockerfile .
- name: Get the version
id: vars
run: echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
- name: Tag docker image
run: docker tag vulcanize/go-ethereum docker.pkg.github.com/vulcanize/go-ethereum/go-ethereum:${{steps.vars.outputs.sha}}
- name: Docker Login
run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
- name: Docker Push
run: docker push docker.pkg.github.com/vulcanize/go-ethereum/go-ethereum:${{steps.vars.outputs.sha}}

66
.github/workflows/on-pr.yml vendored Normal file
View File

@ -0,0 +1,66 @@
name: Build and test
on: [pull_request]
jobs:
build:
name: Run docker build
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Run docker build
run: docker build -t vulcanize/go-ethereum .
geth-unit-test:
name: Run geth unit test
strategy:
matrix:
go-version: [ 1.16.x]
platform: [ubuntu-latest]
runs-on: ${{ matrix.platform }}
env:
GO111MODULE: on
GOPATH: /tmp/go
steps:
- name: Create GOPATH
run: mkdir -p /tmp/go
- name: Install Go
uses: actions/setup-go@v2
with:
go-version: ${{ matrix.go-version }}
- name: Checkout code
uses: actions/checkout@v2
- name: Run unit tests
run: |
make test
statediff-unit-test:
name: Run state diff unit test
env:
GOPATH: /tmp/go
strategy:
matrix:
go-version: [ 1.16.x]
platform: [ubuntu-latest]
runs-on: ${{ matrix.platform }}
steps:
- name: Create GOPATH
run: mkdir -p /tmp/go
- name: Install Go
uses: actions/setup-go@v2
with:
go-version: ${{ matrix.go-version }}
- name: Checkout code
uses: actions/checkout@v2
- name: Start database
run: docker-compose -f docker-compose.yml up -d ipld-eth-db
- name: Run unit tests
run:
make statedifftest

41
.github/workflows/publish.yaml vendored Normal file
View File

@ -0,0 +1,41 @@
name: Publish geth to release
on:
release:
types: [published]
jobs:
push_to_registries:
name: Publish assets to Release
runs-on: ubuntu-latest
steps:
- name: Get the version
id: vars
run: |
echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
echo ::set-output name=tag::$(echo ${GITHUB_REF#refs/tags/})
- name: Docker Login to Github Registry
run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
- name: Docker Pull
run: docker pull docker.pkg.github.com/vulcanize/go-ethereum/go-ethereum:${{steps.vars.outputs.sha}}
- name: Copy ethereum binary file
run: docker run --rm --entrypoint cat docker.pkg.github.com/vulcanize/go-ethereum/go-ethereum:${{steps.vars.outputs.sha}} /usr/local/bin/geth > geth-linux-amd64
- name: Docker Login to Docker Registry
run: echo ${{ secrets.VULCANIZEJENKINS_PAT }} | docker login -u vulcanizejenkins --password-stdin
- name: Tag docker image
run: docker tag docker.pkg.github.com/vulcanize/go-ethereum/go-ethereum:${{steps.vars.outputs.sha}} vulcanize/vdb-geth:${{steps.vars.outputs.tag}}
- name: Docker Push to Docker Hub
run: docker push vulcanize/vdb-geth:${{steps.vars.outputs.tag}}
- name: Get release
id: get_release
uses: bruceadams/get-release@v1.2.0
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Upload Release Asset
id: upload-release-asset
uses: actions/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.get_release.outputs.upload_url }}
asset_path: geth-linux-amd64
asset_name: geth-linux-amd64
asset_content_type: application/octet-stream

247
.travis.yml
View File

@ -1,247 +0,0 @@
language: go
go_import_path: github.com/ethereum/go-ethereum
sudo: false
jobs:
allow_failures:
- stage: build
os: osx
go: 1.17.x
env:
- azure-osx
- azure-ios
- cocoapods-ios
include:
# This builder only tests code linters on latest version of Go
- stage: lint
os: linux
dist: bionic
go: 1.17.x
env:
- lint
git:
submodules: false # avoid cloning ethereum/tests
script:
- go run build/ci.go lint
# These builders create the Docker sub-images for multi-arch push and each
# will attempt to push the multi-arch image if they are the last builder
- stage: build
if: type = push
os: linux
arch: amd64
dist: bionic
go: 1.17.x
env:
- docker
services:
- docker
git:
submodules: false # avoid cloning ethereum/tests
before_install:
- export DOCKER_CLI_EXPERIMENTAL=enabled
script:
- go run build/ci.go docker -image -manifest amd64,arm64 -upload ethereum/client-go
- stage: build
if: type = push
os: linux
arch: arm64
dist: bionic
go: 1.17.x
env:
- docker
services:
- docker
git:
submodules: false # avoid cloning ethereum/tests
before_install:
- export DOCKER_CLI_EXPERIMENTAL=enabled
script:
- go run build/ci.go docker -image -manifest amd64,arm64 -upload ethereum/client-go
# This builder does the Ubuntu PPA upload
- stage: build
if: type = push
os: linux
dist: bionic
go: 1.17.x
env:
- ubuntu-ppa
- GO111MODULE=on
git:
submodules: false # avoid cloning ethereum/tests
addons:
apt:
packages:
- devscripts
- debhelper
- dput
- fakeroot
- python-bzrlib
- python-paramiko
script:
- echo '|1|7SiYPr9xl3uctzovOTj4gMwAC1M=|t6ReES75Bo/PxlOPJ6/GsGbTrM0= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA0aKz5UTUndYgIGG7dQBV+HaeuEZJ2xPHo2DS2iSKvUL4xNMSAY4UguNW+pX56nAQmZKIZZ8MaEvSj6zMEDiq6HFfn5JcTlM80UwlnyKe8B8p7Nk06PPQLrnmQt5fh0HmEcZx+JU9TZsfCHPnX7MNz4ELfZE6cFsclClrKim3BHUIGq//t93DllB+h4O9LHjEUsQ1Sr63irDLSutkLJD6RXchjROXkNirlcNVHH/jwLWR5RcYilNX7S5bIkK8NlWPjsn/8Ua5O7I9/YoE97PpO6i73DTGLh5H9JN/SITwCKBkgSDWUt61uPK3Y11Gty7o2lWsBjhBUm2Y38CBsoGmBw==' >> ~/.ssh/known_hosts
- go run build/ci.go debsrc -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>"
# This builder does the Linux Azure uploads
- stage: build
if: type = push
os: linux
dist: bionic
sudo: required
go: 1.17.x
env:
- azure-linux
- GO111MODULE=on
git:
submodules: false # avoid cloning ethereum/tests
addons:
apt:
packages:
- gcc-multilib
script:
# Build for the primary platforms that Trusty can manage
- go run build/ci.go install -dlgo
- go run build/ci.go archive -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds
- go run build/ci.go install -dlgo -arch 386
- go run build/ci.go archive -arch 386 -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds
# Switch over GCC to cross compilation (breaks 386, hence why do it here only)
- sudo -E apt-get -yq --no-install-suggests --no-install-recommends --force-yes install gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-arm-linux-gnueabihf libc6-dev-armhf-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross
- sudo ln -s /usr/include/asm-generic /usr/include/asm
- GOARM=5 go run build/ci.go install -dlgo -arch arm -cc arm-linux-gnueabi-gcc
- GOARM=5 go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds
- GOARM=6 go run build/ci.go install -dlgo -arch arm -cc arm-linux-gnueabi-gcc
- GOARM=6 go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds
- GOARM=7 go run build/ci.go install -dlgo -arch arm -cc arm-linux-gnueabihf-gcc
- GOARM=7 go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds
- go run build/ci.go install -dlgo -arch arm64 -cc aarch64-linux-gnu-gcc
- go run build/ci.go archive -arch arm64 -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds
# This builder does the Android Maven and Azure uploads
- stage: build
if: type = push
os: linux
dist: bionic
addons:
apt:
packages:
- openjdk-8-jdk
env:
- azure-android
- maven-android
- GO111MODULE=on
git:
submodules: false # avoid cloning ethereum/tests
before_install:
# Install Android and it's dependencies manually, Travis is stale
- export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
- curl https://dl.google.com/android/repository/commandlinetools-linux-6858069_latest.zip -o android.zip
- unzip -q android.zip -d $HOME/sdk && rm android.zip
- mv $HOME/sdk/cmdline-tools $HOME/sdk/latest && mkdir $HOME/sdk/cmdline-tools && mv $HOME/sdk/latest $HOME/sdk/cmdline-tools
- export PATH=$PATH:$HOME/sdk/cmdline-tools/latest/bin
- export ANDROID_HOME=$HOME/sdk
- yes | sdkmanager --licenses >/dev/null
- sdkmanager "platform-tools" "platforms;android-15" "platforms;android-19" "platforms;android-24" "ndk-bundle"
# Install Go to allow building with
- curl https://dl.google.com/go/go1.16.linux-amd64.tar.gz | tar -xz
- export PATH=`pwd`/go/bin:$PATH
- export GOROOT=`pwd`/go
- export GOPATH=$HOME/go
script:
# Build the Android archive and upload it to Maven Central and Azure
- mkdir -p $GOPATH/src/github.com/ethereum
- ln -s `pwd` $GOPATH/src/github.com/ethereum/go-ethereum
- go run build/ci.go aar -signer ANDROID_SIGNING_KEY -signify SIGNIFY_KEY -deploy https://oss.sonatype.org -upload gethstore/builds
# This builder does the OSX Azure, iOS CocoaPods and iOS Azure uploads
- stage: build
if: type = push
os: osx
go: 1.17.x
env:
- azure-osx
- azure-ios
- cocoapods-ios
- GO111MODULE=on
git:
submodules: false # avoid cloning ethereum/tests
script:
- go run build/ci.go install -dlgo
- go run build/ci.go archive -type tar -signer OSX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds
# Build the iOS framework and upload it to CocoaPods and Azure
- gem uninstall cocoapods -a -x
- gem install cocoapods
- mv ~/.cocoapods/repos/master ~/.cocoapods/repos/master.bak
- sed -i '.bak' 's/repo.join/!repo.join/g' $(dirname `gem which cocoapods`)/cocoapods/sources_manager.rb
- if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then git clone --depth=1 https://github.com/CocoaPods/Specs.git ~/.cocoapods/repos/master && pod setup --verbose; fi
- xctool -version
- xcrun simctl list
# Workaround for https://github.com/golang/go/issues/23749
- export CGO_CFLAGS_ALLOW='-fmodules|-fblocks|-fobjc-arc'
- go run build/ci.go xcode -signer IOS_SIGNING_KEY -signify SIGNIFY_KEY -deploy trunk -upload gethstore/builds
# These builders run the tests
- stage: build
os: linux
arch: amd64
dist: bionic
go: 1.17.x
env:
- GO111MODULE=on
script:
- go run build/ci.go test -coverage $TEST_PACKAGES
- stage: build
if: type = pull_request
os: linux
arch: arm64
dist: bionic
go: 1.17.x
env:
- GO111MODULE=on
script:
- go run build/ci.go test -coverage $TEST_PACKAGES
- stage: build
os: linux
dist: bionic
go: 1.16.x
env:
- GO111MODULE=on
script:
- go run build/ci.go test -coverage $TEST_PACKAGES
# This builder does the Azure archive purges to avoid accumulating junk
- stage: build
if: type = cron
os: linux
dist: bionic
go: 1.17.x
env:
- azure-purge
- GO111MODULE=on
git:
submodules: false # avoid cloning ethereum/tests
script:
- go run build/ci.go purge -store gethstore/builds -days 14
# This builder executes race tests
- stage: build
if: type = cron
os: linux
dist: bionic
go: 1.17.x
env:
- GO111MODULE=on
script:
- go run build/ci.go test -race -coverage $TEST_PACKAGES

7
Dockerfile.amd64 Normal file
View File

@ -0,0 +1,7 @@
# Build Geth in a stock Go builder container
FROM golang:1.15.5 as builder
#RUN apk add --no-cache make gcc musl-dev linux-headers git
ADD . /go-ethereum
RUN cd /go-ethereum && make geth

26
Makefile
View File

@ -4,10 +4,31 @@
.PHONY: geth android ios evm all test clean
BIN = $(GOPATH)/bin
## Migration tool
GOOSE = $(BIN)/goose
$(BIN)/goose:
go get -u github.com/pressly/goose/cmd/goose
GOBIN = ./build/bin
GO ?= latest
GORUN = env GO111MODULE=on go run
#Database
HOST_NAME = localhost
PORT = 5432
USER = vdbm
PASSWORD = password
# Set env variable
# `PGPASSWORD` is used by `createdb` and `dropdb`
export PGPASSWORD=$(PASSWORD)
#Test
TEST_DB = vulcanize_public
TEST_CONNECT_STRING = postgresql://$(USER):$(PASSWORD)@$(HOST_NAME):$(PORT)/$(TEST_DB)?sslmode=disable
geth:
$(GORUN) build/ci.go install ./cmd/geth
@echo "Done building."
@ -28,6 +49,11 @@ ios:
@echo "Done building."
@echo "Import \"$(GOBIN)/Geth.framework\" to use the library."
.PHONY: statedifftest
statedifftest: | $(GOOSE)
MODE=statediff go test ./statediff/... -v
test: all
$(GORUN) build/ci.go test

1
build/ci.go
View File

@ -59,6 +59,7 @@ import (
"time"
"github.com/cespare/cp"
"github.com/ethereum/go-ethereum/crypto/signify"
"github.com/ethereum/go-ethereum/internal/build"
"github.com/ethereum/go-ethereum/params"

52
cmd/geth/config.go
View File

@ -25,6 +25,8 @@ import (
"reflect"
"unicode"
"github.com/ethereum/go-ethereum/eth/downloader"
"github.com/ethereum/go-ethereum/statediff"
"gopkg.in/urfave/cli.v1"
"github.com/ethereum/go-ethereum/accounts/external"
@ -149,6 +151,9 @@ func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) {
cfg.Ethstats.URL = ctx.GlobalString(utils.EthStatsURLFlag.Name)
}
applyMetricConfig(ctx, &cfg)
if ctx.GlobalBool(utils.StateDiffFlag.Name) {
cfg.Eth.Diffing = true
}
return stack, cfg
}
@ -159,6 +164,11 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) {
if ctx.GlobalIsSet(utils.OverrideArrowGlacierFlag.Name) {
cfg.Eth.OverrideArrowGlacier = new(big.Int).SetUint64(ctx.GlobalUint64(utils.OverrideArrowGlacierFlag.Name))
}
if cfg.Eth.SyncMode == downloader.LightSync {
return makeLightNode(ctx, stack, cfg)
}
backend, eth := utils.RegisterEthService(stack, &cfg.Eth)
// Configure catalyst.
@ -171,6 +181,34 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) {
}
}
if ctx.GlobalBool(utils.StateDiffFlag.Name) {
var dbParams *statediff.DBParams
if ctx.GlobalIsSet(utils.StateDiffDBFlag.Name) {
dbParams = new(statediff.DBParams)
dbParams.ConnectionURL = ctx.GlobalString(utils.StateDiffDBFlag.Name)
if ctx.GlobalIsSet(utils.StateDiffDBNodeIDFlag.Name) {
dbParams.ID = ctx.GlobalString(utils.StateDiffDBNodeIDFlag.Name)
} else {
utils.Fatalf("Must specify node ID for statediff DB output")
}
if ctx.GlobalIsSet(utils.StateDiffDBClientNameFlag.Name) {
dbParams.ClientName = ctx.GlobalString(utils.StateDiffDBClientNameFlag.Name)
} else {
utils.Fatalf("Must specify client name for statediff DB output")
}
} else {
if ctx.GlobalBool(utils.StateDiffWritingFlag.Name) {
utils.Fatalf("Must pass DB parameters if enabling statediff write loop")
}
}
p := statediff.ServiceParams{
DBParams: dbParams,
EnableWriteLoop: ctx.GlobalBool(utils.StateDiffWritingFlag.Name),
NumWorkers: ctx.GlobalUint(utils.StateDiffWorkersFlag.Name),
}
utils.RegisterStateDiffService(stack, eth, &cfg.Eth, p)
}
// Configure GraphQL if requested
if ctx.GlobalIsSet(utils.GraphQLEnabledFlag.Name) {
utils.RegisterGraphQLService(stack, backend, cfg.Node)
@ -182,6 +220,20 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) {
return stack, backend
}
func makeLightNode(ctx *cli.Context, stack *node.Node, cfg gethConfig) (*node.Node, ethapi.Backend) {
backend := utils.RegisterLesEthService(stack, &cfg.Eth)
// Configure GraphQL if requested
if ctx.GlobalIsSet(utils.GraphQLEnabledFlag.Name) {
utils.RegisterGraphQLService(stack, backend.ApiBackend, cfg.Node)
}
// Add the Ethereum Stats daemon if requested.
if cfg.Ethstats.URL != "" {
utils.RegisterEthStatsService(stack, backend.ApiBackend, cfg.Ethstats.URL)
}
return stack, backend.ApiBackend
}
// dumpConfig is the dumpconfig command.
func dumpConfig(ctx *cli.Context) error {
_, cfg := makeConfigNode(ctx)

6
cmd/geth/main.go
View File

@ -155,6 +155,12 @@ var (
utils.GpoMaxGasPriceFlag,
utils.GpoIgnoreGasPriceFlag,
utils.MinerNotifyFullFlag,
utils.StateDiffFlag,
utils.StateDiffDBFlag,
utils.StateDiffDBNodeIDFlag,
utils.StateDiffDBClientNameFlag,
utils.StateDiffWritingFlag,
utils.StateDiffWorkersFlag,
configFileFlag,
utils.CatalystFlag,
}

11
cmd/geth/usage.go
View File

@ -223,6 +223,17 @@ var AppHelpFlagGroups = []flags.FlagGroup{
utils.NoUSBFlag,
},
},
{
Name: "STATE DIFF",
Flags: []cli.Flag{
utils.StateDiffFlag,
utils.StateDiffDBFlag,
utils.StateDiffDBNodeIDFlag,
utils.StateDiffDBClientNameFlag,
utils.StateDiffWritingFlag,
utils.StateDiffWorkersFlag,
},
},
{
Name: "MISC",
Flags: []cli.Flag{

46
cmd/utils/flags.go
View File

@ -66,6 +66,8 @@ import (
"github.com/ethereum/go-ethereum/p2p/nat"
"github.com/ethereum/go-ethereum/p2p/netutil"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/statediff"
pcsclite "github.com/gballet/go-libpcsclite"
gopsutil "github.com/shirou/gopsutil/mem"
"gopkg.in/urfave/cli.v1"
@ -789,6 +791,30 @@ var (
Name: "catalyst",
Usage: "Catalyst mode (eth2 integration testing)",
}
StateDiffFlag = cli.BoolFlag{
Name: "statediff",
Usage: "Enables the processing of state diffs between each block",
}
StateDiffDBFlag = cli.StringFlag{
Name: "statediff.db",
Usage: "PostgreSQL database connection string for writing state diffs",
}
StateDiffDBNodeIDFlag = cli.StringFlag{
Name: "statediff.dbnodeid",
Usage: "Node ID to use when writing state diffs to database",
}
StateDiffDBClientNameFlag = cli.StringFlag{
Name: "statediff.dbclientname",
Usage: "Client name to use when writing state diffs to database",
}
StateDiffWritingFlag = cli.BoolFlag{
Name: "statediff.writing",
Usage: "Activates progressive writing of state diffs to database as new block are synced",
}
StateDiffWorkersFlag = cli.UintFlag{
Name: "statediff.workers",
Usage: "Number of concurrent workers to use during statediff processing (0 = 1)",
}
)
// MakeDataDir retrieves the currently requested data directory, terminating
@ -1005,6 +1031,10 @@ func setWS(ctx *cli.Context, cfg *node.Config) {
if ctx.GlobalIsSet(WSPathPrefixFlag.Name) {
cfg.WSPathPrefix = ctx.GlobalString(WSPathPrefixFlag.Name)
}
if ctx.GlobalBool(StateDiffFlag.Name) {
cfg.WSModules = append(cfg.WSModules, "statediff")
}
}
// setIPC creates an IPC path configuration from the set command line flags,
@ -1728,6 +1758,15 @@ func RegisterEthService(stack *node.Node, cfg *ethconfig.Config) (ethapi.Backend
return backend.APIBackend, backend
}
// RegisterLesEthService adds an Ethereum les client to the stack.
func RegisterLesEthService(stack *node.Node, cfg *eth.Config) *les.LightEthereum {
backend, err := les.New(stack, cfg)
if err != nil {
Fatalf("Failed to register the Ethereum service: %v", err)
}
return backend
}
// RegisterEthStatsService configures the Ethereum Stats daemon and adds it to
// the given node.
func RegisterEthStatsService(stack *node.Node, backend ethapi.Backend, url string) {
@ -1743,6 +1782,13 @@ func RegisterGraphQLService(stack *node.Node, backend ethapi.Backend, cfg node.C
}
}
// RegisterStateDiffService configures and registers a service to stream state diff data over RPC
func RegisterStateDiffService(stack *node.Node, ethServ *eth.Ethereum, cfg *ethconfig.Config, params statediff.ServiceParams) {
if err := statediff.New(stack, ethServ, cfg, params); err != nil {
Fatalf("Failed to register the Statediff service: %v", err)
}
}
func SetupMetrics(ctx *cli.Context) {
if metrics.Enabled {
log.Info("Enabling metrics collection")

50
core/blockchain.go
View File

@ -28,6 +28,8 @@ import (
"sync/atomic"
"time"
lru "github.com/hashicorp/golang-lru"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/mclock"
"github.com/ethereum/go-ethereum/common/prque"
@ -44,7 +46,6 @@ import (
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/trie"
lru "github.com/hashicorp/golang-lru"
)
var (
@ -132,6 +133,7 @@ type CacheConfig struct {
Preimages bool // Whether to store preimage of trie key to the disk
SnapshotWait bool // Wait for snapshot construction on startup. TODO(karalabe): This is a dirty hack for testing, nuke it
StateDiffing bool // Whether or not the statediffing service is running
}
// defaultCacheConfig are the default caching values if none are specified by the
@ -210,6 +212,10 @@ type BlockChain struct {
processor Processor // Block transaction processor interface
vmConfig vm.Config
// Locked roots and their mutex
trieLock sync.Mutex
lockedRoots map[common.Hash]bool
shouldPreserve func(*types.Block) bool // Function used to determine whether should preserve the given block.
}
@ -248,6 +254,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par
futureBlocks: futureBlocks,
engine: engine,
vmConfig: vmConfig,
lockedRoots: make(map[common.Hash]bool),
}
bc.validator = NewBlockValidator(chainConfig, bc, engine)
bc.prefetcher = newStatePrefetcher(chainConfig, bc, engine)
@ -823,7 +830,10 @@ func (bc *BlockChain) Stop() {
}
}
for !bc.triegc.Empty() {
triedb.Dereference(bc.triegc.PopItem().(common.Hash))
pruneRoot := bc.triegc.PopItem().(common.Hash)
if !bc.TrieLocked(pruneRoot) {
triedb.Dereference(pruneRoot)
}
}
if size, _ := triedb.Size(); size != 0 {
log.Error("Dangling trie nodes after full cleanup")
@ -1236,6 +1246,11 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
triedb.Reference(root, common.Hash{}) // metadata reference to keep trie alive
bc.triegc.Push(root, -int64(block.NumberU64()))
// If we are statediffing, lock the trie until the statediffing service is done using it
if bc.cacheConfig.StateDiffing {
bc.LockTrie(root)
}
if current := block.NumberU64(); current > TriesInMemory {
// If we exceeded our memory allowance, flush matured singleton nodes to disk
var (
@ -1274,7 +1289,11 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
bc.triegc.Push(root, number)
break
}
triedb.Dereference(root.(common.Hash))
pruneRoot := root.(common.Hash)
if !bc.TrieLocked(pruneRoot) {
log.Debug("Dereferencing", "root", root.(common.Hash).Hex())
triedb.Dereference(pruneRoot)
}
}
}
}
@ -2188,3 +2207,28 @@ func (bc *BlockChain) InsertHeaderChain(chain []*types.Header, checkFreq int) (i
_, err := bc.hc.InsertHeaderChain(chain, start)
return 0, err
}
// TrieLocked returns whether the trie associated with the provided root is locked for use
func (bc *BlockChain) TrieLocked(root common.Hash) bool {
bc.trieLock.Lock()
locked, ok := bc.lockedRoots[root]
bc.trieLock.Unlock()
if !ok {
return false
}
return locked
}
// LockTrie prevents dereferencing of the provided root
func (bc *BlockChain) LockTrie(root common.Hash) {
bc.trieLock.Lock()
bc.lockedRoots[root] = true
bc.trieLock.Unlock()
}
// UnlockTrie allows dereferencing of the provided root- provided it was previously locked
func (bc *BlockChain) UnlockTrie(root common.Hash) {
bc.trieLock.Lock()
bc.lockedRoots[root] = false
bc.trieLock.Unlock()
}

4
core/types/receipt.go
View File

@ -70,6 +70,7 @@ type Receipt struct {
BlockHash common.Hash `json:"blockHash,omitempty"`
BlockNumber *big.Int `json:"blockNumber,omitempty"`
TransactionIndex uint `json:"transactionIndex"`
LogRoot common.Hash `json:"logRoot"`
}
type receiptMarshaling struct {
@ -136,6 +137,9 @@ func NewReceipt(root []byte, failed bool, cumulativeGasUsed uint64) *Receipt {
// EncodeRLP implements rlp.Encoder, and flattens the consensus fields of a receipt
// into an RLP stream. If no post state is present, byzantium fork is assumed.
// For a legacy Receipt this returns RLP([PostStateOrStatus, CumulativeGasUsed, Bloom, Logs])
// For a EIP-2718 Receipt this returns RLP(TxType || ReceiptPayload)
// For a EIP-2930 Receipt, TxType == 0x01 and ReceiptPayload == RLP([PostStateOrStatus, CumulativeGasUsed, Bloom, Logs])
func (r *Receipt) EncodeRLP(w io.Writer) error {
data := &receiptRLP{r.statusEncoding(), r.CumulativeGasUsed, r.Bloom, r.Logs}
if r.Type == LegacyTxType {

10
core/types/transaction.go
View File

@ -88,6 +88,9 @@ type TxData interface {
}
// EncodeRLP implements rlp.Encoder
// For a legacy Transaction this returns RLP([AccountNonce, GasPrice, GasLimit, Recipient, Amount, Data, V, R, S])
// For a EIP-2718 Transaction this returns RLP(TxType || TxPayload)
// For a EIP-2930 Transaction, TxType == 0x01 and TxPayload == RLP([ChainID, AccountNonce, GasPrice, GasLimit, Recipient, Amount, Data, AccessList, V, R, S]
func (tx *Transaction) EncodeRLP(w io.Writer) error {
if tx.Type() == LegacyTxType {
return rlp.Encode(w, tx.inner)
@ -108,9 +111,10 @@ func (tx *Transaction) encodeTyped(w *bytes.Buffer) error {
return rlp.Encode(w, tx.inner)
}
// MarshalBinary returns the canonical encoding of the transaction.
// For legacy transactions, it returns the RLP encoding. For EIP-2718 typed
// transactions, it returns the type and payload.
// MarshalBinary returns the canonical consensus encoding of the transaction.
// For a legacy Transaction this returns RLP([AccountNonce, GasPrice, GasLimit, Recipient, Amount, Data, V, R, S])
// For a EIP-2718 Transaction this returns TxType || TxPayload
// For a EIP-2930 Transaction, TxType == 0x01 and TxPayload == RLP([ChainID, AccountNonce, GasPrice, GasLimit, Recipient, Amount, Data, AccessList, V, R, S]
func (tx *Transaction) MarshalBinary() ([]byte, error) {
if tx.Type() == LegacyTxType {
return rlp.EncodeToBytes(tx.inner)

17
docker-compose.yml Normal file
View File

@ -0,0 +1,17 @@
version: '3.2'
services:
ipld-eth-db:
restart: always
image: vulcanize/ipld-eth-db:v0.2.0
environment:
POSTGRES_USER: "vdbm"
POSTGRES_DB: "vulcanize_public"
POSTGRES_PASSWORD: "password"
volumes:
- geth_node:/var/lib/postgresql/data
ports:
- "127.0.0.1:5432:5432"
volumes:
geth_node:

1
eth/backend.go
View File

@ -186,6 +186,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
TrieTimeLimit: config.TrieTimeout,
SnapshotLimit: config.SnapshotCache,
Preimages: config.Preimages,
StateDiffing: config.Diffing,
}
)
eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, chainConfig, eth.engine, vmConfig, eth.shouldPreserve, &config.TxLookupLimit)

4
eth/ethconfig/config.go
View File

@ -204,6 +204,10 @@ type Config struct {
// Arrow Glacier block override (TODO: remove after the fork)
OverrideArrowGlacier *big.Int `toml:",omitempty"`
// Signify whether or not we are producing statediffs
// If we are, do not dereference state roots until the statediffing service is done with them
Diffing bool
}
// CreateConsensusEngine creates a consensus engine for the given chain configuration.

13
go.mod
View File

@ -41,13 +41,22 @@ require (
github.com/influxdata/influxdb v1.8.3
github.com/influxdata/influxdb-client-go/v2 v2.4.0
github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 // indirect
github.com/ipfs/go-block-format v0.0.2
github.com/ipfs/go-cid v0.0.7
github.com/ipfs/go-ipfs-blockstore v1.0.1
github.com/ipfs/go-ipfs-ds-help v1.0.0
github.com/ipfs/go-ipld-format v0.2.0
github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458
github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e
github.com/jmoiron/sqlx v1.2.0
github.com/julienschmidt/httprouter v1.2.0
github.com/karalabe/usb v0.0.0-20211005121534-4c5740d64559
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/lib/pq v1.10.2
github.com/mattn/go-colorable v0.1.8
github.com/mattn/go-isatty v0.0.12
github.com/mattn/go-sqlite3 v1.14.7 // indirect
github.com/multiformats/go-multihash v0.0.14
github.com/naoina/go-stringutil v0.1.0 // indirect
github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416
github.com/olekukonko/tablewriter v0.0.5
@ -61,12 +70,12 @@ require (
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
github.com/tklauser/go-sysconf v0.3.5 // indirect
github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect
golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912
golang.org/x/text v0.3.6
golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba
golang.org/x/tools v0.1.2 // indirect
gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce
gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6
gopkg.in/urfave/cli.v1 v1.20.0

96
go.sum
View File

@ -161,11 +161,15 @@ github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34
github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU=
github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI=
@ -208,6 +212,7 @@ github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OI
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.5 h1:kxhtnfFVi+rYdOALN0B3k9UT86zVJKfBimRaciULW4I=
github.com/google/uuid v1.1.5/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
@ -218,10 +223,13 @@ github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0U
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29 h1:sezaKhEfPFg8W0Enm61B9Gs911H8iesGY5R8NDPtd1M=
github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc=
github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU=
github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48=
github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE=
github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs=
github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao=
@ -248,13 +256,42 @@ github.com/influxdata/promql/v2 v2.12.0/go.mod h1:fxOPu+DY0bqCTCECchSRtWfc+0X19y
github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6/go.mod h1:bSgUQ7q5ZLSO+bKBGqJiCBGAl+9DxyW63zLTujjUlOE=
github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mqiSBE6Ffsg94weZZ2c+v/ciT8QRHFOap7EKDrR0=
github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1:Wbbw6tYNvwa5dlB6304Sd+82Z3f7PmVZHVKU637d4po=
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
github.com/ipfs/go-block-format v0.0.2 h1:qPDvcP19izTjU8rgo6p7gTXZlkMkF5bz5G3fqIsSCPE=
github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=
github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog=
github.com/ipfs/go-cid v0.0.7 h1:ysQJVJA3fNDF1qigJbsSQOdjhVLsOEoPdh0+R97k3jY=
github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I=
github.com/ipfs/go-datastore v0.4.1/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA=
github.com/ipfs/go-datastore v0.4.2 h1:h8/n7WPzhp239kkLws+epN3Ic7YtcBPgcaXfEfdVDWM=
github.com/ipfs/go-datastore v0.4.2/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA=
github.com/ipfs/go-ipfs-blockstore v1.0.1 h1:fnuVj4XdZp4yExhd0CnUwAiMNJHiPnfInhiuwz4lW1w=
github.com/ipfs/go-ipfs-blockstore v1.0.1/go.mod h1:MGNZlHNEnR4KGgPHM3/k8lBySIOK2Ve+0KjZubKlaOE=
github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=
github.com/ipfs/go-ipfs-ds-help v1.0.0 h1:bEQ8hMGs80h0sR8O4tfDgV6B01aaF9qeTrujrTLYV3g=
github.com/ipfs/go-ipfs-ds-help v1.0.0/go.mod h1:ujAbkeIgkKAWtxxNkoZHWLCyk5JpPoKnGyCcsoF6ueE=
github.com/ipfs/go-ipfs-util v0.0.1 h1:Wz9bL2wB2YBJqggkA4dD7oSmqB4cAnpNbGrlHJulv50=
github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc=
github.com/ipfs/go-ipld-format v0.2.0 h1:xGlJKkArkmBvowr+GMCX0FEZtkro71K1AwiKnL37mwA=
github.com/ipfs/go-ipld-format v0.2.0/go.mod h1:3l3C1uKoadTPbeNfrDi+xMInYKlx2Cvg1BuydPSdzQs=
github.com/ipfs/go-log v0.0.1 h1:9XTUN/rW64BCG1YhPK9Hoy3q8nr4gOmHHBpgFdfw6Lc=
github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM=
github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg=
github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY=
github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 h1:6OvNmYgJyexcZ3pYbTI9jWx5tHo1Dee/tWbLMfPe2TA=
github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8 h1:bspPhN+oKYFk5fcGNuQzp6IGzYQSenLEgH3s6jkXrWw=
github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY=
github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e h1:UvSe12bq+Uj2hWd8aOlwPmoZ+CITRFrdit+sDGfAg8U=
github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU=
github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA=
github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
@ -267,6 +304,7 @@ github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E
github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0=
github.com/karalabe/usb v0.0.0-20211005121534-4c5740d64559 h1:0VWDXPNE0brOek1Q8bLfzKkvOzwbQE/snjGojlCr8CY=
github.com/karalabe/usb v0.0.0-20211005121534-4c5740d64559/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU=
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
@ -291,10 +329,14 @@ github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL
github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8=
github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM=
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
@ -303,6 +345,7 @@ github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HN
github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d h1:oNAwILwmgWKFpuU+dXvI6dl9jG2mAWAZLX3r9s0PPiw=
github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc=
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
@ -310,16 +353,40 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky
github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/mattn/go-sqlite3 v1.14.7 h1:fxWBnXkxfM6sRiuH3bqJ4CfzZojMOLVc0UTsTglEghA=
github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g=
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ=
github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771 h1:MHkK1uRtFbVqvAgvWxafZe54+5uBxLluGylDiKgdhwo=
github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag=
github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/pointerstructure v1.2.0 h1:O+i9nHnXS3l/9Wu7r4NrEdwA2VFTicjUEN1uBnDo34A=
github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8=
github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc=
github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg=
github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI=
github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA=
github.com/multiformats/go-base36 v0.1.0 h1:JR6TyF7JjGd3m6FbLU2cOxhC0Li8z8dLNGQ89tUg4F4=
github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM=
github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs=
github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk=
github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc=
github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U=
github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
github.com/multiformats/go-multihash v0.0.14 h1:QoBceQYQQtNUuf6s7wHxnE2c8bhbMqhfGzNI032se/I=
github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
github.com/multiformats/go-varint v0.0.5 h1:XVZwSo04Cs3j/jS0uAEPpT3JY6DzMcVLLoWOSnCxOjg=
github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hzifhks=
github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0=
@ -385,6 +452,8 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
@ -412,27 +481,35 @@ github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/X
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc h1:9lDbC6Rz4bwmou+oE6Dt4Cb2BGMur5eR/GYptkKUVHo=
github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM=
github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk=
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w=
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a h1:kr2P4QFmQr29mSLA43kwrOcgcReGTfbE9N577tCTuBc=
golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@ -453,6 +530,7 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl
golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNTEKDvWeuc1yieZ8qUzUE=
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
@ -460,6 +538,7 @@ golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKG
golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -467,6 +546,7 @@ golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73r
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@ -483,8 +563,8 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY
golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI=
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@ -506,6 +586,7 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@ -534,8 +615,9 @@ golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912 h1:uCLL3g5wH2xjxVREVuAbP9JM5PPKjRbXKRa6IBjkzmU=
golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
@ -554,6 +636,7 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb
golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE=
golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@ -573,6 +656,7 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@ -581,6 +665,8 @@ golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapK
golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200108203644-89082a384178/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA=
golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@ -603,6 +689,7 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@ -633,6 +720,7 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=

1
mobile/android_test.go
View File

@ -155,6 +155,7 @@ public class AndroidTest extends InstrumentationTestCase {
//
// This method has been adapted from golang.org/x/mobile/bind/java/seq_test.go/runTest
func TestAndroid(t *testing.T) {
t.Skip("Skipping this test for statediff as this is not relevant")
// Skip tests on Windows altogether
if runtime.GOOS == "windows" {
t.Skip("cannot test Android bindings on Windows, skipping")

8
params/version.go
View File

@ -21,10 +21,10 @@ import (
)
const (
VersionMajor = 1 // Major version component of the current release
VersionMinor = 10 // Minor version component of the current release
VersionPatch = 13 // Patch version component of the current release
VersionMeta = "stable" // Version metadata to append to the version string
VersionMajor = 1 // Major version component of the current release
VersionMinor = 10 // Minor version component of the current release
VersionPatch = 13 // Patch version component of the current release
VersionMeta = "statediff-0.0.28" // Version metadata to append to the version string
)
// Version holds the textual version string.

2
rpc/http.go
View File

@ -32,7 +32,7 @@ import (
)
const (
maxRequestContentLength = 1024 * 1024 * 5
maxRequestContentLength = 1024 * 1024 * 12
contentType = "application/json"
)

266
statediff/README.md Normal file
View File

@ -0,0 +1,266 @@
# Statediff
This package provides an auxiliary service that asynchronously processes state diff objects from chain events,
either relaying the state objects to RPC subscribers or writing them directly to Postgres as IPLD objects.
It also exposes RPC endpoints for fetching or writing to Postgres the state diff at a specific block height
or for a specific block hash, this operates on historical block and state data and so depends on a complete state archive.
Data is emitted in this differential format in order to make it feasible to IPLD-ize and index the *entire* Ethereum state
(including intermediate state and storage trie nodes). If this state diff process is ran continuously from genesis,
the entire state at any block can be materialized from the cumulative differentials up to that point.
## Statediff object
A state diff `StateObject` is the collection of all the state and storage trie nodes that have been updated in a given block.
For convenience, we also associate these nodes with the block number and hash, and optionally the set of code hashes and code for any
contracts deployed in this block.
A complete state diff `StateObject` will include all state and storage intermediate nodes, which is necessary for generating proofs and for
traversing the tries.
```go
// StateObject is a collection of state (and linked storage nodes) as well as the associated block number, block hash,
// and a set of code hashes and their code
type StateObject struct {
BlockNumber *big.Int `json:"blockNumber" gencodec:"required"`
BlockHash common.Hash `json:"blockHash" gencodec:"required"`
Nodes []StateNode `json:"nodes" gencodec:"required"`
CodeAndCodeHashes []CodeAndCodeHash `json:"codeMapping"`
}
// StateNode holds the data for a single state diff node
type StateNode struct {
NodeType NodeType `json:"nodeType" gencodec:"required"`
Path []byte `json:"path" gencodec:"required"`
NodeValue []byte `json:"value" gencodec:"required"`
StorageNodes []StorageNode `json:"storage"`
LeafKey []byte `json:"leafKey"`
}
// StorageNode holds the data for a single storage diff node
type StorageNode struct {
NodeType NodeType `json:"nodeType" gencodec:"required"`
Path []byte `json:"path" gencodec:"required"`
NodeValue []byte `json:"value" gencodec:"required"`
LeafKey []byte `json:"leafKey"`
}
// CodeAndCodeHash struct for holding codehash => code mappings
// we can't use an actual map because they are not rlp serializable
type CodeAndCodeHash struct {
Hash common.Hash `json:"codeHash"`
Code []byte `json:"code"`
}
```
These objects are packed into a `Payload` structure which can additionally associate the `StateObject`
with the block (header, uncles, and transactions), receipts, and total difficulty.
This `Payload` encapsulates all of the differential data at a given block, and allows us to index the entire Ethereum data structure
as hash-linked IPLD objects.
```go
// Payload packages the data to send to state diff subscriptions
type Payload struct {
BlockRlp []byte `json:"blockRlp"`
TotalDifficulty *big.Int `json:"totalDifficulty"`
ReceiptsRlp []byte `json:"receiptsRlp"`
StateObjectRlp []byte `json:"stateObjectRlp" gencodec:"required"`
encoded []byte
err error
}
```
## Usage
This state diffing service runs as an auxiliary service concurrent to the regular syncing process of the geth node.
### CLI configuration
This service introduces a CLI flag namespace `statediff`
- `--statediff` flag is used to turn on the service
- `--statediff.writing` is used to tell the service to write state diff objects it produces from synced ChainEvents directly to a configured Postgres database
- `--statediff.workers` is used to set the number of concurrent workers to process state diff objects and write them into the database
- `--statediff.db` is the connection string for the Postgres database to write to
- `--statediff.dbnodeid` is the node id to use in the Postgres database
- `--statediff.dbclientname` is the client name to use in the Postgres database
The service can only operate in full sync mode (`--syncmode=full`), but only the historical RPC endpoints require an archive node (`--gcmode=archive`)
e.g.
`
./build/bin/geth --syncmode=full --gcmode=archive --statediff --statediff.writing --statediff.db="postgres://localhost:5432/vulcanize_testing?sslmode=disable" --statediff.db.init=true --statediff.dbnodeid={nodeId} --statediff.dbclientname={dbClientName}
`
### RPC endpoints
The state diffing service exposes both a WS subscription endpoint, and a number of HTTP unary endpoints.
Each of these endpoints requires a set of parameters provided by the caller
```go
// Params is used to carry in parameters from subscribing/requesting clients configuration
type Params struct {
IntermediateStateNodes bool
IntermediateStorageNodes bool
IncludeBlock bool
IncludeReceipts bool
IncludeTD bool
IncludeCode bool
WatchedAddresses []common.Address
WatchedStorageSlots []common.Hash
}
```
Using these params we can tell the service whether to include state and/or storage intermediate nodes; whether
to include the associated block (header, uncles, and transactions); whether to include the associated receipts;
whether to include the total difficulty for this block; whether to include the set of code hashes and code for
contracts deployed in this block; whether to limit the diffing process to a list of specific addresses; and/or
whether to limit the diffing process to a list of specific storage slot keys.
#### Subscription endpoint
A websocket supporting RPC endpoint is exposed for subscribing to state diff `StateObjects` that come off the head of the chain while the geth node syncs.
```go
// Stream is a subscription endpoint that fires off state diff payloads as they are created
Stream(ctx context.Context, params Params) (*rpc.Subscription, error)
```
To expose this endpoint the node needs to have the websocket server turned on (`--ws`),
and the `statediff` namespace exposed (`--ws.api=statediff`).
Go code subscriptions to this endpoint can be created using the `rpc.Client.Subscribe()` method,
with the "statediff" namespace, a `statediff.Payload` channel, and the name of the statediff api's rpc method: "stream".
e.g.
```go
cli, err := rpc.Dial("ipcPathOrWsURL")
if err != nil {
// handle error
}
stateDiffPayloadChan := make(chan statediff.Payload, 20000)
methodName := "stream"
params := statediff.Params{
IncludeBlock: true,
IncludeTD: true,
IncludeReceipts: true,
IntermediateStorageNodes: true,
IntermediateStateNodes: true,
}
rpcSub, err := cli.Subscribe(context.Background(), statediff.APIName, stateDiffPayloadChan, methodName, params)
if err != nil {
// handle error
}
for {
select {
case stateDiffPayload := <- stateDiffPayloadChan:
// process the payload
case err := <- rpcSub.Err():
// handle rpc subscription error
}
}
```
#### Unary endpoints
The service also exposes unary RPC endpoints for retrieving the state diff `StateObject` for a specific block height/hash.
```go
// StateDiffAt returns a state diff payload at the specific blockheight
StateDiffAt(ctx context.Context, blockNumber uint64, params Params) (*Payload, error)
// StateDiffFor returns a state diff payload for the specific blockhash
StateDiffFor(ctx context.Context, blockHash common.Hash, params Params) (*Payload, error)
```
To expose this endpoint the node needs to have the HTTP server turned on (`--http`),
and the `statediff` namespace exposed (`--http.api=statediff`).
### Direct indexing into Postgres
If `--statediff.writing` is set, the service will convert the state diff `StateObject` data into IPLD objects, persist them directly to Postgres,
and generate secondary indexes around the IPLD data.
The schema and migrations for this Postgres database are provided in [vulcanize/statediff-migrations](https://github.com/vulcanize/statediff-migrations).
#### Postgres setup
We use [pressly/goose](https://github.com/pressly/goose) as our Postgres migration manager.
You can also load the Postgres schema directly into a database using
`psql database_name < schema.sql`
This will only work on a version 12.4 Postgres database.
See [vulcanize/statediff-migrations](https://github.com/vulcanize/statediff-migrations) for more info.
#### Schema overview
Our Postgres schemas are built around a single IPFS backing Postgres IPLD blockstore table (`public.blocks`) that conforms with [go-ds-sql](https://github.com/ipfs/go-ds-sql/blob/master/postgres/postgres.go).
All IPLD objects are stored in this table, where `key` is the blockstore-prefixed multihash key for the IPLD object and `data` contains
the bytes for the IPLD block (in the case of all Ethereum IPLDs, this is the RLP byte encoding of the Ethereum object).
The IPLD objects in this table can be traversed using an IPLD DAG interface, but since this table only maps multihash to raw IPLD object
it is not particularly useful for searching through the data by looking up Ethereum objects by their constituent fields
(e.g. by block number, tx source/recipient, state/storage trie node path). To improve the accessibility of these objects
we create an Ethereum [advanced data layout](https://github.com/ipld/specs#schemas-and-advanced-data-layouts) (ADL) by generating secondary
indexes on top of the raw IPLDs in other Postgres tables.
These secondary index tables fall under the `eth` schema and follow an `{objectType}_cids` naming convention.
These tables provide a view into individual fields of the underlying Ethereum IPLD objects, allowing lookups on these fields, and reference the raw IPLD objects stored in `public.blocks`
by foreign keys to their multihash keys.
Additionally, these tables maintain the hash-linked nature of Ethereum objects to one another. E.g. a storage trie node entry in the `storage_cids`
table contains a `state_id` foreign key which references the `id` for the `state_cids` entry that contains the state leaf node for the contract that storage node belongs to,
and in turn that `state_cids` entry contains a `header_id` foreign key which references the `id` of the `header_cids` entry that contains the header for the block these state and storage nodes were updated (diffed).
### Optimization
On mainnet this process is extremely IO intensive and requires significant resources to allow it to keep up with the head of the chain.
The state diff processing time for a specific block is dependent on the number and complexity of the state changes that occur in a block and
the number of updated state nodes that are available in the in-memory cache vs must be retrieved from disc.
If memory permits, one means of improving the efficiency of this process is to increase the in-memory trie cache allocation.
This can be done by increasing the overall `--cache` allocation and/or by increasing the % of the cache allocated to trie
usage with `--cache.trie`.
## Versioning, Branches, Rebasing, and Releasing
Internal tagged releases are maintained for building the latest version of statediffing geth or using it as a go mod dependency.
When a new core go-ethereum version is released, statediffing geth is rebased onto and adjusted to work with the new tag.
We want to maintain a complete record of our git history, but in order to make frequent and timely rebases feasible we also
need to be able to squash our work before performing a rebase. To this end we retain multiple branches with partial incremental history that culminate in
the full incremental history.
### Versioning
Versioning for of statediffing geth follows the below format:
`{Root Version}-statediff-{Statediff Version}`
Where "root version" is the version of the tagged release from the core go-ethereum repository that our release is rebased on top of
and "statediff version" is the version tracking the state of the statediffing service code.
E.g. the version at the time of writing this is v1.10.3-statediff-0.0.23, v0.0.23 of the statediffing code rebased on top of the v1.10.3 core tag.
The statediff version is included in the `VersionMeta` in params/version.go
### Branches
We maintain two official kinds of branches:
Major Branch: `{Root Version}-statediff`
Major branches retain the cumulative state of all changes made before the latest root version rebase and track the full incremental history of changes made between the latest root version rebase and the next.
Aside from creating the branch by performing the rebase described in the section below, these branches are never worked off of or committed to directly.
Feature Branch: `{Root Version}-statediff-{Statediff Version}`
Feature branches are checked out from a major branch in order to work on a new feature or fix for the statediffing code.
The statediff version of a feature branch is the new version it affects on the major branch when merged. Internal tagged releases
are cut against these branches after they are merged back to the major branch.
If a developer is unsure what version their patch should affect, they should remain working on an unofficial branch. From there
they can open a PR against the targeted root branch and be directed to the appropriate feature version and branch.
### Rebasing
When a new root tagged release comes out we rebase our statediffing code on top of the new tag using the following process:
1. Checkout a new major branch for the tag from the current major branch
2. On the new major branch, squash all our commits since the last major rebase
3. On the new major branch, perform the rebase against the new tag
4. Push the new major branch to the remote
5. From the new major branch, checkout a new feature branch based on the new major version and the last statediff version
6. On this new feature branch, add the new major branch to the .github/workflows/on-master.yml list of "on push" branches
7. On this new feature branch, make any fixes/adjustments required for all statediffing geth tests to pass
8. PR this feature branch into the new major branch, this PR will trigger CI tests and builds.
9. After merging PR, rebase feature branch onto major branch
10. Cut a new release targeting the feature branch, this release should have the new root version but the same statediff version as the last release

151
statediff/api.go Normal file
View File

@ -0,0 +1,151 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package statediff
import (
"context"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rpc"
. "github.com/ethereum/go-ethereum/statediff/types"
)
// APIName is the namespace used for the state diffing service API
const APIName = "statediff"
// APIVersion is the version of the state diffing service API
const APIVersion = "0.0.1"
// PublicStateDiffAPI provides an RPC subscription interface
// that can be used to stream out state diffs as they
// are produced by a full node
type PublicStateDiffAPI struct {
sds IService
}
// NewPublicStateDiffAPI creates an rpc subscription interface for the underlying statediff service
func NewPublicStateDiffAPI(sds IService) *PublicStateDiffAPI {
return &PublicStateDiffAPI{
sds: sds,
}
}
// Stream is the public method to setup a subscription that fires off statediff service payloads as they are created
func (api *PublicStateDiffAPI) Stream(ctx context.Context, params Params) (*rpc.Subscription, error) {
// ensure that the RPC connection supports subscriptions
notifier, supported := rpc.NotifierFromContext(ctx)
if !supported {
return nil, rpc.ErrNotificationsUnsupported
}
// create subscription and start waiting for events
rpcSub := notifier.CreateSubscription()
go func() {
// subscribe to events from the statediff service
payloadChannel := make(chan Payload, chainEventChanSize)
quitChan := make(chan bool, 1)
api.sds.Subscribe(rpcSub.ID, payloadChannel, quitChan, params)
// loop and await payloads and relay them to the subscriber with the notifier
for {
select {
case payload := <-payloadChannel:
if err := notifier.Notify(rpcSub.ID, payload); err != nil {
log.Error("Failed to send state diff packet; error: " + err.Error())
if err := api.sds.Unsubscribe(rpcSub.ID); err != nil {
log.Error("Failed to unsubscribe from the state diff service; error: " + err.Error())
}
return
}
case err := <-rpcSub.Err():
if err != nil {
log.Error("State diff service rpcSub error: " + err.Error())
err = api.sds.Unsubscribe(rpcSub.ID)
if err != nil {
log.Error("Failed to unsubscribe from the state diff service; error: " + err.Error())
}
return
}
case <-quitChan:
// don't need to unsubscribe, service does so before sending the quit signal
return
}
}
}()
return rpcSub, nil
}
// StateDiffAt returns a state diff payload at the specific blockheight
func (api *PublicStateDiffAPI) StateDiffAt(ctx context.Context, blockNumber uint64, params Params) (*Payload, error) {
return api.sds.StateDiffAt(blockNumber, params)
}
// StateDiffFor returns a state diff payload for the specific blockhash
func (api *PublicStateDiffAPI) StateDiffFor(ctx context.Context, blockHash common.Hash, params Params) (*Payload, error) {
return api.sds.StateDiffFor(blockHash, params)
}
// StateTrieAt returns a state trie payload at the specific blockheight
func (api *PublicStateDiffAPI) StateTrieAt(ctx context.Context, blockNumber uint64, params Params) (*Payload, error) {
return api.sds.StateTrieAt(blockNumber, params)
}
// StreamCodeAndCodeHash writes all of the codehash=>code pairs out to a websocket channel
func (api *PublicStateDiffAPI) StreamCodeAndCodeHash(ctx context.Context, blockNumber uint64) (*rpc.Subscription, error) {
// ensure that the RPC connection supports subscriptions
notifier, supported := rpc.NotifierFromContext(ctx)
if !supported {
return nil, rpc.ErrNotificationsUnsupported
}
// create subscription and start waiting for events
rpcSub := notifier.CreateSubscription()
payloadChan := make(chan CodeAndCodeHash, chainEventChanSize)
quitChan := make(chan bool)
api.sds.StreamCodeAndCodeHash(blockNumber, payloadChan, quitChan)
go func() {
for {
select {
case payload := <-payloadChan:
if err := notifier.Notify(rpcSub.ID, payload); err != nil {
log.Error("Failed to send code and codehash packet", "err", err)
return
}
case err := <-rpcSub.Err():
log.Error("State diff service rpcSub error", "err", err)
return
case <-quitChan:
return
}
}
}()
return rpcSub, nil
}
// WriteStateDiffAt writes a state diff object directly to DB at the specific blockheight
func (api *PublicStateDiffAPI) WriteStateDiffAt(ctx context.Context, blockNumber uint64, params Params) error {
return api.sds.WriteStateDiffAt(blockNumber, params)
}
// WriteStateDiffFor writes a state diff object directly to DB for the specific block hash
func (api *PublicStateDiffAPI) WriteStateDiffFor(ctx context.Context, blockHash common.Hash, params Params) error {
return api.sds.WriteStateDiffFor(blockHash, params)
}

747
statediff/builder.go Normal file
View File

@ -0,0 +1,747 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// Contains a batch of utility type declarations used by the tests. As the node
// operates on unique types, a lot of them are needed to check various features.
package statediff
import (
"bytes"
"fmt"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rlp"
sdtrie "github.com/ethereum/go-ethereum/statediff/trie"
. "github.com/ethereum/go-ethereum/statediff/types"
"github.com/ethereum/go-ethereum/trie"
)
var (
nullHashBytes = common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000000")
emptyNode, _ = rlp.EncodeToBytes([]byte{})
emptyContractRoot = crypto.Keccak256Hash(emptyNode)
nullCodeHash = crypto.Keccak256Hash([]byte{}).Bytes()
)
// Builder interface exposes the method for building a state diff between two blocks
type Builder interface {
BuildStateDiffObject(args Args, params Params) (StateObject, error)
BuildStateTrieObject(current *types.Block) (StateObject, error)
WriteStateDiffObject(args StateRoots, params Params, output StateNodeSink, codeOutput CodeSink) error
}
type builder struct {
stateCache state.Database
}
// convenience
func stateNodeAppender(nodes *[]StateNode) StateNodeSink {
return func(node StateNode) error {
*nodes = append(*nodes, node)
return nil
}
}
func storageNodeAppender(nodes *[]StorageNode) StorageNodeSink {
return func(node StorageNode) error {
*nodes = append(*nodes, node)
return nil
}
}
func codeMappingAppender(codeAndCodeHashes *[]CodeAndCodeHash) CodeSink {
return func(c CodeAndCodeHash) error {
*codeAndCodeHashes = append(*codeAndCodeHashes, c)
return nil
}
}
// NewBuilder is used to create a statediff builder
func NewBuilder(stateCache state.Database) Builder {
return &builder{
stateCache: stateCache, // state cache is safe for concurrent reads
}
}
// BuildStateTrieObject builds a state trie object from the provided block
func (sdb *builder) BuildStateTrieObject(current *types.Block) (StateObject, error) {
currentTrie, err := sdb.stateCache.OpenTrie(current.Root())
if err != nil {
return StateObject{}, fmt.Errorf("error creating trie for block %d: %v", current.Number(), err)
}
it := currentTrie.NodeIterator([]byte{})
stateNodes, codeAndCodeHashes, err := sdb.buildStateTrie(it)
if err != nil {
return StateObject{}, fmt.Errorf("error collecting state nodes for block %d: %v", current.Number(), err)
}
return StateObject{
BlockNumber: current.Number(),
BlockHash: current.Hash(),
Nodes: stateNodes,
CodeAndCodeHashes: codeAndCodeHashes,
}, nil
}
func (sdb *builder) buildStateTrie(it trie.NodeIterator) ([]StateNode, []CodeAndCodeHash, error) {
stateNodes := make([]StateNode, 0)
codeAndCodeHashes := make([]CodeAndCodeHash, 0)
for it.Next(true) {
// skip value nodes
if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
continue
}
node, nodeElements, err := sdtrie.ResolveNode(it, sdb.stateCache.TrieDB())
if err != nil {
return nil, nil, err
}
switch node.NodeType {
case Leaf:
var account types.StateAccount
if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil {
return nil, nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", node.Path, err)
}
partialPath := trie.CompactToHex(nodeElements[0].([]byte))
valueNodePath := append(node.Path, partialPath...)
encodedPath := trie.HexToCompact(valueNodePath)
leafKey := encodedPath[1:]
node.LeafKey = leafKey
if !bytes.Equal(account.CodeHash, nullCodeHash) {
var storageNodes []StorageNode
err := sdb.buildStorageNodesEventual(account.Root, nil, true, storageNodeAppender(&storageNodes))
if err != nil {
return nil, nil, fmt.Errorf("failed building eventual storage diffs for account %+v\r\nerror: %v", account, err)
}
node.StorageNodes = storageNodes
// emit codehash => code mappings for cod
codeHash := common.BytesToHash(account.CodeHash)
code, err := sdb.stateCache.ContractCode(common.Hash{}, codeHash)
if err != nil {
return nil, nil, fmt.Errorf("failed to retrieve code for codehash %s\r\n error: %v", codeHash.String(), err)
}
codeAndCodeHashes = append(codeAndCodeHashes, CodeAndCodeHash{
Hash: codeHash,
Code: code,
})
}
stateNodes = append(stateNodes, node)
case Extension, Branch:
stateNodes = append(stateNodes, node)
default:
return nil, nil, fmt.Errorf("unexpected node type %s", node.NodeType)
}
}
return stateNodes, codeAndCodeHashes, it.Error()
}
// BuildStateDiffObject builds a statediff object from two blocks and the provided parameters
func (sdb *builder) BuildStateDiffObject(args Args, params Params) (StateObject, error) {
var stateNodes []StateNode
var codeAndCodeHashes []CodeAndCodeHash
err := sdb.WriteStateDiffObject(
StateRoots{OldStateRoot: args.OldStateRoot, NewStateRoot: args.NewStateRoot},
params, stateNodeAppender(&stateNodes), codeMappingAppender(&codeAndCodeHashes))
if err != nil {
return StateObject{}, err
}
return StateObject{
BlockHash: args.BlockHash,
BlockNumber: args.BlockNumber,
Nodes: stateNodes,
CodeAndCodeHashes: codeAndCodeHashes,
}, nil
}
// Writes a statediff object to output callback
func (sdb *builder) WriteStateDiffObject(args StateRoots, params Params, output StateNodeSink, codeOutput CodeSink) error {
if !params.IntermediateStateNodes || len(params.WatchedAddresses) > 0 {
// if we are watching only specific accounts then we are only diffing leaf nodes
return sdb.buildStateDiffWithoutIntermediateStateNodes(args, params, output, codeOutput)
} else {
return sdb.buildStateDiffWithIntermediateStateNodes(args, params, output, codeOutput)
}
}
func (sdb *builder) buildStateDiffWithIntermediateStateNodes(args StateRoots, params Params, output StateNodeSink, codeOutput CodeSink) error {
// Load tries for old and new states
oldTrie, err := sdb.stateCache.OpenTrie(args.OldStateRoot)
if err != nil {
return fmt.Errorf("error creating trie for oldStateRoot: %v", err)
}
newTrie, err := sdb.stateCache.OpenTrie(args.NewStateRoot)
if err != nil {
return fmt.Errorf("error creating trie for newStateRoot: %v", err)
}
// collect a slice of all the intermediate nodes that were touched and exist at B
// a map of their leafkey to all the accounts that were touched and exist at B
// and a slice of all the paths for the nodes in both of the above sets
diffAccountsAtB, diffPathsAtB, err := sdb.createdAndUpdatedStateWithIntermediateNodes(
oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}),
output)
if err != nil {
return fmt.Errorf("error collecting createdAndUpdatedNodes: %v", err)
}
// collect a slice of all the nodes that existed at a path in A that doesn't exist in B
// a map of their leafkey to all the accounts that were touched and exist at A
diffAccountsAtA, err := sdb.deletedOrUpdatedState(
oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}),
diffPathsAtB, output)
if err != nil {
return fmt.Errorf("error collecting deletedOrUpdatedNodes: %v", err)
}
// collect and sort the leafkey keys for both account mappings into a slice
createKeys := sortKeys(diffAccountsAtB)
deleteKeys := sortKeys(diffAccountsAtA)
// and then find the intersection of these keys
// these are the leafkeys for the accounts which exist at both A and B but are different
// this also mutates the passed in createKeys and deleteKeys, removing the intersection keys
// and leaving the truly created or deleted keys in place
updatedKeys := findIntersection(createKeys, deleteKeys)
// build the diff nodes for the updated accounts using the mappings at both A and B as directed by the keys found as the intersection of the two
err = sdb.buildAccountUpdates(
diffAccountsAtB, diffAccountsAtA, updatedKeys,
params.WatchedStorageSlots, params.IntermediateStorageNodes, output)
if err != nil {
return fmt.Errorf("error building diff for updated accounts: %v", err)
}
// build the diff nodes for created accounts
err = sdb.buildAccountCreations(diffAccountsAtB, params.WatchedStorageSlots, params.IntermediateStorageNodes, output, codeOutput)
if err != nil {
return fmt.Errorf("error building diff for created accounts: %v", err)
}
return nil
}
func (sdb *builder) buildStateDiffWithoutIntermediateStateNodes(args StateRoots, params Params, output StateNodeSink, codeOutput CodeSink) error {
// Load tries for old (A) and new (B) states
oldTrie, err := sdb.stateCache.OpenTrie(args.OldStateRoot)
if err != nil {
return fmt.Errorf("error creating trie for oldStateRoot: %v", err)
}
newTrie, err := sdb.stateCache.OpenTrie(args.NewStateRoot)
if err != nil {
return fmt.Errorf("error creating trie for newStateRoot: %v", err)
}
// collect a map of their leafkey to all the accounts that were touched and exist at B
// and a slice of all the paths for the nodes in both of the above sets
diffAccountsAtB, diffPathsAtB, err := sdb.createdAndUpdatedState(
oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}),
params.WatchedAddresses)
if err != nil {
return fmt.Errorf("error collecting createdAndUpdatedNodes: %v", err)
}
// collect a slice of all the nodes that existed at a path in A that doesn't exist in B
// a map of their leafkey to all the accounts that were touched and exist at A
diffAccountsAtA, err := sdb.deletedOrUpdatedState(
oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}),
diffPathsAtB, output)
if err != nil {
return fmt.Errorf("error collecting deletedOrUpdatedNodes: %v", err)
}
// collect and sort the leafkeys for both account mappings into a slice
createKeys := sortKeys(diffAccountsAtB)
deleteKeys := sortKeys(diffAccountsAtA)
// and then find the intersection of these keys
// these are the leafkeys for the accounts which exist at both A and B but are different
// this also mutates the passed in createKeys and deleteKeys, removing in intersection keys
// and leaving the truly created or deleted keys in place
updatedKeys := findIntersection(createKeys, deleteKeys)
// build the diff nodes for the updated accounts using the mappings at both A and B as directed by the keys found as the intersection of the two
err = sdb.buildAccountUpdates(
diffAccountsAtB, diffAccountsAtA, updatedKeys,
params.WatchedStorageSlots, params.IntermediateStorageNodes, output)
if err != nil {
return fmt.Errorf("error building diff for updated accounts: %v", err)
}
// build the diff nodes for created accounts
err = sdb.buildAccountCreations(diffAccountsAtB, params.WatchedStorageSlots, params.IntermediateStorageNodes, output, codeOutput)
if err != nil {
return fmt.Errorf("error building diff for created accounts: %v", err)
}
return nil
}
// createdAndUpdatedState returns
// a mapping of their leafkeys to all the accounts that exist in a different state at B than A
// and a slice of the paths for all of the nodes included in both
func (sdb *builder) createdAndUpdatedState(a, b trie.NodeIterator, watchedAddresses []common.Address) (AccountMap, map[string]bool, error) {
diffPathsAtB := make(map[string]bool)
diffAcountsAtB := make(AccountMap)
it, _ := trie.NewDifferenceIterator(a, b)
for it.Next(true) {
// skip value nodes
if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
continue
}
node, nodeElements, err := sdtrie.ResolveNode(it, sdb.stateCache.TrieDB())
if err != nil {
return nil, nil, err
}
if node.NodeType == Leaf {
// created vs updated is important for leaf nodes since we need to diff their storage
// so we need to map all changed accounts at B to their leafkey, since account can change pathes but not leafkey
var account types.StateAccount
if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil {
return nil, nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", node.Path, err)
}
partialPath := trie.CompactToHex(nodeElements[0].([]byte))
valueNodePath := append(node.Path, partialPath...)
encodedPath := trie.HexToCompact(valueNodePath)
leafKey := encodedPath[1:]
if isWatchedAddress(watchedAddresses, leafKey) {
diffAcountsAtB[common.Bytes2Hex(leafKey)] = accountWrapper{
NodeType: node.NodeType,
Path: node.Path,
NodeValue: node.NodeValue,
LeafKey: leafKey,
Account: &account,
}
}
}
// add both intermediate and leaf node paths to the list of diffPathsAtB
diffPathsAtB[common.Bytes2Hex(node.Path)] = true
}
return diffAcountsAtB, diffPathsAtB, it.Error()
}
// createdAndUpdatedStateWithIntermediateNodes returns
// a slice of all the intermediate nodes that exist in a different state at B than A
// a mapping of their leafkeys to all the accounts that exist in a different state at B than A
// and a slice of the paths for all of the nodes included in both
func (sdb *builder) createdAndUpdatedStateWithIntermediateNodes(a, b trie.NodeIterator, output StateNodeSink) (AccountMap, map[string]bool, error) {
diffPathsAtB := make(map[string]bool)
diffAcountsAtB := make(AccountMap)
it, _ := trie.NewDifferenceIterator(a, b)
for it.Next(true) {
// skip value nodes
if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
continue
}
node, nodeElements, err := sdtrie.ResolveNode(it, sdb.stateCache.TrieDB())
if err != nil {
return nil, nil, err
}
switch node.NodeType {
case Leaf:
// created vs updated is important for leaf nodes since we need to diff their storage
// so we need to map all changed accounts at B to their leafkey, since account can change paths but not leafkey
var account types.StateAccount
if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil {
return nil, nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", node.Path, err)
}
partialPath := trie.CompactToHex(nodeElements[0].([]byte))
valueNodePath := append(node.Path, partialPath...)
encodedPath := trie.HexToCompact(valueNodePath)
leafKey := encodedPath[1:]
diffAcountsAtB[common.Bytes2Hex(leafKey)] = accountWrapper{
NodeType: node.NodeType,
Path: node.Path,
NodeValue: node.NodeValue,
LeafKey: leafKey,
Account: &account,
}
case Extension, Branch:
// create a diff for any intermediate node that has changed at b
// created vs updated makes no difference for intermediate nodes since we do not need to diff storage
if err := output(StateNode{
NodeType: node.NodeType,
Path: node.Path,
NodeValue: node.NodeValue,
}); err != nil {
return nil, nil, err
}
default:
return nil, nil, fmt.Errorf("unexpected node type %s", node.NodeType)
}
// add both intermediate and leaf node paths to the list of diffPathsAtB
diffPathsAtB[common.Bytes2Hex(node.Path)] = true
}
return diffAcountsAtB, diffPathsAtB, it.Error()
}
// deletedOrUpdatedState returns a slice of all the pathes that are emptied at B
// and a mapping of their leafkeys to all the accounts that exist in a different state at A than B
func (sdb *builder) deletedOrUpdatedState(a, b trie.NodeIterator, diffPathsAtB map[string]bool, output StateNodeSink) (AccountMap, error) {
diffAccountAtA := make(AccountMap)
it, _ := trie.NewDifferenceIterator(b, a)
for it.Next(true) {
// skip value nodes
if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
continue
}
node, nodeElements, err := sdtrie.ResolveNode(it, sdb.stateCache.TrieDB())
if err != nil {
return nil, err
}
switch node.NodeType {
case Leaf:
// map all different accounts at A to their leafkey
var account types.StateAccount
if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil {
return nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", node.Path, err)
}
partialPath := trie.CompactToHex(nodeElements[0].([]byte))
valueNodePath := append(node.Path, partialPath...)
encodedPath := trie.HexToCompact(valueNodePath)
leafKey := encodedPath[1:]
diffAccountAtA[common.Bytes2Hex(leafKey)] = accountWrapper{
NodeType: node.NodeType,
Path: node.Path,
NodeValue: node.NodeValue,
LeafKey: leafKey,
Account: &account,
}
// if this node's path did not show up in diffPathsAtB
// that means the node at this path was deleted (or moved) in B
// emit an empty "removed" diff to signify as such
if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; !ok {
if err := output(StateNode{
Path: node.Path,
NodeValue: []byte{},
NodeType: Removed,
LeafKey: leafKey,
}); err != nil {
return nil, err
}
}
case Extension, Branch:
// if this node's path did not show up in diffPathsAtB
// that means the node at this path was deleted (or moved) in B
// emit an empty "removed" diff to signify as such
if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; !ok {
if err := output(StateNode{
Path: node.Path,
NodeValue: []byte{},
NodeType: Removed,
}); err != nil {
return nil, err
}
}
// fall through, we did everything we need to do with these node types
default:
return nil, fmt.Errorf("unexpected node type %s", node.NodeType)
}
}
return diffAccountAtA, it.Error()
}
// buildAccountUpdates uses the account diffs maps for A => B and B => A and the known intersection of their leafkeys
// to generate the statediff node objects for all of the accounts that existed at both A and B but in different states
// needs to be called before building account creations and deletions as this mutates
// those account maps to remove the accounts which were updated
func (sdb *builder) buildAccountUpdates(creations, deletions AccountMap, updatedKeys []string,
watchedStorageKeys []common.Hash, intermediateStorageNodes bool, output StateNodeSink) error {
var err error
for _, key := range updatedKeys {
createdAcc := creations[key]
deletedAcc := deletions[key]
var storageDiffs []StorageNode
if deletedAcc.Account != nil && createdAcc.Account != nil {
oldSR := deletedAcc.Account.Root
newSR := createdAcc.Account.Root
err = sdb.buildStorageNodesIncremental(
oldSR, newSR, watchedStorageKeys, intermediateStorageNodes,
storageNodeAppender(&storageDiffs))
if err != nil {
return fmt.Errorf("failed building incremental storage diffs for account with leafkey %s\r\nerror: %v", key, err)
}
}
if err = output(StateNode{
NodeType: createdAcc.NodeType,
Path: createdAcc.Path,
NodeValue: createdAcc.NodeValue,
LeafKey: createdAcc.LeafKey,
StorageNodes: storageDiffs,
}); err != nil {
return err
}
delete(creations, key)
delete(deletions, key)
}
return nil
}
// buildAccountCreations returns the statediff node objects for all the accounts that exist at B but not at A
// it also returns the code and codehash for created contract accounts
func (sdb *builder) buildAccountCreations(accounts AccountMap, watchedStorageKeys []common.Hash, intermediateStorageNodes bool, output StateNodeSink, codeOutput CodeSink) error {
for _, val := range accounts {
diff := StateNode{
NodeType: val.NodeType,
Path: val.Path,
LeafKey: val.LeafKey,
NodeValue: val.NodeValue,
}
if !bytes.Equal(val.Account.CodeHash, nullCodeHash) {
// For contract creations, any storage node contained is a diff
var storageDiffs []StorageNode
err := sdb.buildStorageNodesEventual(val.Account.Root, watchedStorageKeys, intermediateStorageNodes, storageNodeAppender(&storageDiffs))
if err != nil {
return fmt.Errorf("failed building eventual storage diffs for node %x\r\nerror: %v", val.Path, err)
}
diff.StorageNodes = storageDiffs
// emit codehash => code mappings for cod
codeHash := common.BytesToHash(val.Account.CodeHash)
code, err := sdb.stateCache.ContractCode(common.Hash{}, codeHash)
if err != nil {
return fmt.Errorf("failed to retrieve code for codehash %s\r\n error: %v", codeHash.String(), err)
}
if err := codeOutput(CodeAndCodeHash{
Hash: codeHash,
Code: code,
}); err != nil {
return err
}
}
if err := output(diff); err != nil {
return err
}
}
return nil
}
// buildStorageNodesEventual builds the storage diff node objects for a created account
// i.e. it returns all the storage nodes at this state, since there is no previous state
func (sdb *builder) buildStorageNodesEventual(sr common.Hash, watchedStorageKeys []common.Hash, intermediateNodes bool, output StorageNodeSink) error {
if bytes.Equal(sr.Bytes(), emptyContractRoot.Bytes()) {
return nil
}
log.Debug("Storage Root For Eventual Diff", "root", sr.Hex())
sTrie, err := sdb.stateCache.OpenTrie(sr)
if err != nil {
log.Info("error in build storage diff eventual", "error", err)
return err
}
it := sTrie.NodeIterator(make([]byte, 0))
err = sdb.buildStorageNodesFromTrie(it, watchedStorageKeys, intermediateNodes, output)
if err != nil {
return err
}
return nil
}
// buildStorageNodesFromTrie returns all the storage diff node objects in the provided node interator
// if any storage keys are provided it will only return those leaf nodes
// including intermediate nodes can be turned on or off
func (sdb *builder) buildStorageNodesFromTrie(it trie.NodeIterator, watchedStorageKeys []common.Hash, intermediateNodes bool, output StorageNodeSink) error {
for it.Next(true) {
// skip value nodes
if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
continue
}
node, nodeElements, err := sdtrie.ResolveNode(it, sdb.stateCache.TrieDB())
if err != nil {
return err
}
switch node.NodeType {
case Leaf:
partialPath := trie.CompactToHex(nodeElements[0].([]byte))
valueNodePath := append(node.Path, partialPath...)
encodedPath := trie.HexToCompact(valueNodePath)
leafKey := encodedPath[1:]
if isWatchedStorageKey(watchedStorageKeys, leafKey) {
if err := output(StorageNode{
NodeType: node.NodeType,
Path: node.Path,
NodeValue: node.NodeValue,
LeafKey: leafKey,
}); err != nil {
return err
}
}
case Extension, Branch:
if intermediateNodes {
if err := output(StorageNode{
NodeType: node.NodeType,
Path: node.Path,
NodeValue: node.NodeValue,
}); err != nil {
return err
}
}
default:
return fmt.Errorf("unexpected node type %s", node.NodeType)
}
}
return it.Error()
}
// buildStorageNodesIncremental builds the storage diff node objects for all nodes that exist in a different state at B than A
func (sdb *builder) buildStorageNodesIncremental(oldSR common.Hash, newSR common.Hash, watchedStorageKeys []common.Hash, intermediateNodes bool, output StorageNodeSink) error {
if bytes.Equal(newSR.Bytes(), oldSR.Bytes()) {
return nil
}
log.Debug("Storage Roots for Incremental Diff", "old", oldSR.Hex(), "new", newSR.Hex())
oldTrie, err := sdb.stateCache.OpenTrie(oldSR)
if err != nil {
return err
}
newTrie, err := sdb.stateCache.OpenTrie(newSR)
if err != nil {
return err
}
diffPathsAtB, err := sdb.createdAndUpdatedStorage(
oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}),
watchedStorageKeys, intermediateNodes, output)
if err != nil {
return err
}
err = sdb.deletedOrUpdatedStorage(oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}),
diffPathsAtB, watchedStorageKeys, intermediateNodes, output)
if err != nil {
return err
}
return nil
}
func (sdb *builder) createdAndUpdatedStorage(a, b trie.NodeIterator, watchedKeys []common.Hash, intermediateNodes bool, output StorageNodeSink) (map[string]bool, error) {
diffPathsAtB := make(map[string]bool)
it, _ := trie.NewDifferenceIterator(a, b)
for it.Next(true) {
// skip value nodes
if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
continue
}
node, nodeElements, err := sdtrie.ResolveNode(it, sdb.stateCache.TrieDB())
if err != nil {
return nil, err
}
switch node.NodeType {
case Leaf:
partialPath := trie.CompactToHex(nodeElements[0].([]byte))
valueNodePath := append(node.Path, partialPath...)
encodedPath := trie.HexToCompact(valueNodePath)
leafKey := encodedPath[1:]
if isWatchedStorageKey(watchedKeys, leafKey) {
if err := output(StorageNode{
NodeType: node.NodeType,
Path: node.Path,
NodeValue: node.NodeValue,
LeafKey: leafKey,
}); err != nil {
return nil, err
}
}
case Extension, Branch:
if intermediateNodes {
if err := output(StorageNode{
NodeType: node.NodeType,
Path: node.Path,
NodeValue: node.NodeValue,
}); err != nil {
return nil, err
}
}
default:
return nil, fmt.Errorf("unexpected node type %s", node.NodeType)
}
diffPathsAtB[common.Bytes2Hex(node.Path)] = true
}
return diffPathsAtB, it.Error()
}
func (sdb *builder) deletedOrUpdatedStorage(a, b trie.NodeIterator, diffPathsAtB map[string]bool, watchedKeys []common.Hash, intermediateNodes bool, output StorageNodeSink) error {
it, _ := trie.NewDifferenceIterator(b, a)
for it.Next(true) {
// skip value nodes
if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
continue
}
node, nodeElements, err := sdtrie.ResolveNode(it, sdb.stateCache.TrieDB())
if err != nil {
return err
}
// if this node path showed up in diffPathsAtB
// that means this node was updated at B and we already have the updated diff for it
// otherwise that means this node was deleted in B and we need to add a "removed" diff to represent that event
if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; ok {
continue
}
switch node.NodeType {
case Leaf:
partialPath := trie.CompactToHex(nodeElements[0].([]byte))
valueNodePath := append(node.Path, partialPath...)
encodedPath := trie.HexToCompact(valueNodePath)
leafKey := encodedPath[1:]
if isWatchedStorageKey(watchedKeys, leafKey) {
if err := output(StorageNode{
NodeType: Removed,
Path: node.Path,
NodeValue: []byte{},
LeafKey: leafKey,
}); err != nil {
return err
}
}
case Extension, Branch:
if intermediateNodes {
if err := output(StorageNode{
NodeType: Removed,
Path: node.Path,
NodeValue: []byte{},
}); err != nil {
return err
}
}
default:
return fmt.Errorf("unexpected node type %s", node.NodeType)
}
}
return it.Error()
}
// isWatchedAddress is used to check if a state account corresponds to one of the addresses the builder is configured to watch
func isWatchedAddress(watchedAddresses []common.Address, stateLeafKey []byte) bool {
// If we aren't watching any specific addresses, we are watching everything
if len(watchedAddresses) == 0 {
return true
}
for _, addr := range watchedAddresses {
addrHashKey := crypto.Keccak256(addr.Bytes())
if bytes.Equal(addrHashKey, stateLeafKey) {
return true
}
}
return false
}
// isWatchedStorageKey is used to check if a storage leaf corresponds to one of the storage slots the builder is configured to watch
func isWatchedStorageKey(watchedKeys []common.Hash, storageLeafKey []byte) bool {
// If we aren't watching any specific addresses, we are watching everything
if len(watchedKeys) == 0 {
return true
}
for _, hashKey := range watchedKeys {
if bytes.Equal(hashKey.Bytes(), storageLeafKey) {
return true
}
}
return false
}

2322
statediff/builder_test.go Normal file
View File

File diff suppressed because it is too large Load Diff

73
statediff/helpers.go Normal file
View File

@ -0,0 +1,73 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// Contains a batch of utility type declarations used by the tests. As the node
// operates on unique types, a lot of them are needed to check various features.
package statediff
import (
"sort"
"strings"
)
func sortKeys(data AccountMap) []string {
keys := make([]string, 0, len(data))
for key := range data {
keys = append(keys, key)
}
sort.Strings(keys)
return keys
}
// findIntersection finds the set of strings from both arrays that are equivalent
// a and b must first be sorted
// this is used to find which keys have been both "deleted" and "created" i.e. they were updated
func findIntersection(a, b []string) []string {
lenA := len(a)
lenB := len(b)
iOfA, iOfB := 0, 0
updates := make([]string, 0)
if iOfA >= lenA || iOfB >= lenB {
return updates
}
for {
switch strings.Compare(a[iOfA], b[iOfB]) {
// -1 when a[iOfA] < b[iOfB]
case -1:
iOfA++
if iOfA >= lenA {
return updates
}
// 0 when a[iOfA] == b[iOfB]
case 0:
updates = append(updates, a[iOfA])
iOfA++
iOfB++
if iOfA >= lenA || iOfB >= lenB {
return updates
}
// 1 when a[iOfA] > b[iOfB]
case 1:
iOfB++
if iOfB >= lenB {
return updates
}
}
}
}

45
statediff/indexer/helpers.go Normal file
View File

@ -0,0 +1,45 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package indexer
import (
"fmt"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/statediff/types"
)
// ResolveFromNodeType wrapper around NodeType.Int() so that we maintain backwards compatibility
func ResolveFromNodeType(nodeType types.NodeType) int {
return nodeType.Int()
}
// ChainConfig returns the appropriate ethereum chain config for the provided chain id
func ChainConfig(chainID uint64) (*params.ChainConfig, error) {
switch chainID {
case 1:
return params.MainnetChainConfig, nil
case 3:
return params.RopstenChainConfig, nil
case 4:
return params.RinkebyChainConfig, nil
case 5:
return params.GoerliChainConfig, nil
default:
return nil, fmt.Errorf("chain config for chainid %d not available", chainID)
}
}

551
statediff/indexer/indexer.go Normal file
View File

@ -0,0 +1,551 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
// Package indexer provides an interface for pushing and indexing IPLD objects into a Postgres database
// Metrics for reporting processing and connection stats are defined in ./metrics.go
package indexer
import (
"fmt"
"math/big"
"time"
"github.com/ipfs/go-cid"
node "github.com/ipfs/go-ipld-format"
"github.com/jmoiron/sqlx"
"github.com/multiformats/go-multihash"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/statediff/indexer/ipfs/ipld"
"github.com/ethereum/go-ethereum/statediff/indexer/models"
"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
)
var (
indexerMetrics = RegisterIndexerMetrics(metrics.DefaultRegistry)
dbMetrics = RegisterDBMetrics(metrics.DefaultRegistry)
)
const (
RemovedNodeStorageCID = "bagmacgzayxjemamg64rtzet6pwznzrydydsqbnstzkbcoo337lmaixmfurya"
RemovedNodeStateCID = "baglacgzayxjemamg64rtzet6pwznzrydydsqbnstzkbcoo337lmaixmfurya"
RemovedNodeMhKey = "/blocks/DMQMLUSGAGDPOIZ4SJ7H3MW4Y4B4BZIAWZJ4VARHHN57VWAELWC2I4A"
)
// Indexer interface to allow substitution of mocks for testing
type Indexer interface {
PushBlock(block *types.Block, receipts types.Receipts, totalDifficulty *big.Int) (*BlockTx, error)
PushStateNode(tx *BlockTx, stateNode sdtypes.StateNode) error
PushCodeAndCodeHash(tx *BlockTx, codeAndCodeHash sdtypes.CodeAndCodeHash) error
ReportDBMetrics(delay time.Duration, quit <-chan bool)
}
// StateDiffIndexer satisfies the Indexer interface for ethereum statediff objects
type StateDiffIndexer struct {
chainConfig *params.ChainConfig
dbWriter *PostgresCIDWriter
init bool
}
// NewStateDiffIndexer creates a pointer to a new PayloadConverter which satisfies the PayloadConverter interface
func NewStateDiffIndexer(chainConfig *params.ChainConfig, db *postgres.DB) (*StateDiffIndexer, error) {
// Write the removed node to the db on init
if err := shared.PublishDirectWithDB(db, RemovedNodeMhKey, []byte{}); err != nil {
return nil, err
}
return &StateDiffIndexer{
chainConfig: chainConfig,
dbWriter: NewPostgresCIDWriter(db),
}, nil
}
type BlockTx struct {
dbtx *sqlx.Tx
BlockNumber uint64
headerID int64
Close func(err error) error
}
// ReportDBMetrics is a reporting function to run as goroutine
func (sdi *StateDiffIndexer) ReportDBMetrics(delay time.Duration, quit <-chan bool) {
if !metrics.Enabled {
return
}
ticker := time.NewTicker(delay)
go func() {
for {
select {
case <-ticker.C:
dbMetrics.Update(sdi.dbWriter.db.Stats())
case <-quit:
ticker.Stop()
return
}
}
}()
}
// PushBlock pushes and indexes block data in database, except state & storage nodes (includes header, uncles, transactions & receipts)
// Returns an initiated DB transaction which must be Closed via defer to commit or rollback
func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receipts, totalDifficulty *big.Int) (*BlockTx, error) {
start, t := time.Now(), time.Now()
blockHash := block.Hash()
blockHashStr := blockHash.String()
height := block.NumberU64()
traceMsg := fmt.Sprintf("indexer stats for statediff at %d with hash %s:\r\n", height, blockHashStr)
transactions := block.Transactions()
// Derive any missing fields
if err := receipts.DeriveFields(sdi.chainConfig, blockHash, height, transactions); err != nil {
return nil, err
}
// Generate the block iplds
headerNode, uncleNodes, txNodes, txTrieNodes, rctNodes, rctTrieNodes, logTrieNodes, logLeafNodeCIDs, rctLeafNodeCIDs, err := ipld.FromBlockAndReceipts(block, receipts)
if err != nil {
return nil, fmt.Errorf("error creating IPLD nodes from block and receipts: %v", err)
}
if len(txNodes) != len(rctNodes) || len(rctNodes) != len(rctLeafNodeCIDs) {
return nil, fmt.Errorf("expected number of transactions (%d), receipts (%d), and receipt trie leaf nodes (%d)to be equal", len(txNodes), len(rctNodes), len(rctLeafNodeCIDs))
}
// Calculate reward
var reward *big.Int
// in PoA networks block reward is 0
if sdi.chainConfig.Clique != nil {
reward = big.NewInt(0)
} else {
reward = CalcEthBlockReward(block.Header(), block.Uncles(), block.Transactions(), receipts)
}
t = time.Now()
// Begin new db tx for everything
tx, err := sdi.dbWriter.db.Beginx()
if err != nil {
return nil, err
}
defer func() {
if p := recover(); p != nil {
shared.Rollback(tx)
panic(p)
} else if err != nil {
shared.Rollback(tx)
}
}()
blockTx := &BlockTx{
dbtx: tx,
// handle transaction commit or rollback for any return case
Close: func(err error) error {
if p := recover(); p != nil {
shared.Rollback(tx)
panic(p)
} else if err != nil {
shared.Rollback(tx)
} else {
tDiff := time.Since(t)
indexerMetrics.tStateStoreCodeProcessing.Update(tDiff)
traceMsg += fmt.Sprintf("state, storage, and code storage processing time: %s\r\n", tDiff.String())
t = time.Now()
err = tx.Commit()
tDiff = time.Since(t)
indexerMetrics.tPostgresCommit.Update(tDiff)
traceMsg += fmt.Sprintf("postgres transaction commit duration: %s\r\n", tDiff.String())
}
traceMsg += fmt.Sprintf(" TOTAL PROCESSING DURATION: %s\r\n", time.Since(start).String())
log.Debug(traceMsg)
return err
},
}
tDiff := time.Since(t)
indexerMetrics.tFreePostgres.Update(tDiff)
traceMsg += fmt.Sprintf("time spent waiting for free postgres tx: %s:\r\n", tDiff.String())
t = time.Now()
// Publish and index header, collect headerID
var headerID int64
headerID, err = sdi.processHeader(tx, block.Header(), headerNode, reward, totalDifficulty)
if err != nil {
return nil, err
}
tDiff = time.Since(t)
indexerMetrics.tHeaderProcessing.Update(tDiff)
traceMsg += fmt.Sprintf("header processing time: %s\r\n", tDiff.String())
t = time.Now()
// Publish and index uncles
err = sdi.processUncles(tx, headerID, height, uncleNodes)
if err != nil {
return nil, err
}
tDiff = time.Since(t)
indexerMetrics.tUncleProcessing.Update(tDiff)
traceMsg += fmt.Sprintf("uncle processing time: %s\r\n", tDiff.String())
t = time.Now()
// Publish and index receipts and txs
err = sdi.processReceiptsAndTxs(tx, processArgs{
headerID: headerID,
blockNumber: block.Number(),
receipts: receipts,
txs: transactions,
rctNodes: rctNodes,
rctTrieNodes: rctTrieNodes,
txNodes: txNodes,
txTrieNodes: txTrieNodes,
logTrieNodes: logTrieNodes,
logLeafNodeCIDs: logLeafNodeCIDs,
rctLeafNodeCIDs: rctLeafNodeCIDs,
})
if err != nil {
return nil, err
}
tDiff = time.Since(t)
indexerMetrics.tTxAndRecProcessing.Update(tDiff)
traceMsg += fmt.Sprintf("tx and receipt processing time: %s\r\n", tDiff.String())
t = time.Now()
blockTx.BlockNumber = height
blockTx.headerID = headerID
return blockTx, err
}
// processHeader publishes and indexes a header IPLD in Postgres
// it returns the headerID
func (sdi *StateDiffIndexer) processHeader(tx *sqlx.Tx, header *types.Header, headerNode node.Node, reward, td *big.Int) (int64, error) {
// publish header
if err := shared.PublishIPLD(tx, headerNode); err != nil {
return 0, fmt.Errorf("error publishing header IPLD: %v", err)
}
var baseFee *int64
if header.BaseFee != nil {
baseFee = new(int64)
*baseFee = header.BaseFee.Int64()
}
// index header
return sdi.dbWriter.upsertHeaderCID(tx, models.HeaderModel{
CID: headerNode.Cid().String(),
MhKey: shared.MultihashKeyFromCID(headerNode.Cid()),
ParentHash: header.ParentHash.String(),
BlockNumber: header.Number.String(),
BlockHash: header.Hash().String(),
TotalDifficulty: td.String(),
Reward: reward.String(),
Bloom: header.Bloom.Bytes(),
StateRoot: header.Root.String(),
RctRoot: header.ReceiptHash.String(),
TxRoot: header.TxHash.String(),
UncleRoot: header.UncleHash.String(),
Timestamp: header.Time,
BaseFee: baseFee,
})
}
// processUncles publishes and indexes uncle IPLDs in Postgres
func (sdi *StateDiffIndexer) processUncles(tx *sqlx.Tx, headerID int64, blockNumber uint64, uncleNodes []*ipld.EthHeader) error {
// publish and index uncles
for _, uncleNode := range uncleNodes {
if err := shared.PublishIPLD(tx, uncleNode); err != nil {
return fmt.Errorf("error publishing uncle IPLD: %v", err)
}
var uncleReward *big.Int
// in PoA networks uncle reward is 0
if sdi.chainConfig.Clique != nil {
uncleReward = big.NewInt(0)
} else {
uncleReward = CalcUncleMinerReward(blockNumber, uncleNode.Number.Uint64())
}
uncle := models.UncleModel{
CID: uncleNode.Cid().String(),
MhKey: shared.MultihashKeyFromCID(uncleNode.Cid()),
ParentHash: uncleNode.ParentHash.String(),
BlockHash: uncleNode.Hash().String(),
Reward: uncleReward.String(),
}
if err := sdi.dbWriter.upsertUncleCID(tx, uncle, headerID); err != nil {
return err
}
}
return nil
}
// processArgs bundles arguments to processReceiptsAndTxs
type processArgs struct {
headerID int64
blockNumber *big.Int
receipts types.Receipts
txs types.Transactions
rctNodes []*ipld.EthReceipt
rctTrieNodes []*ipld.EthRctTrie
txNodes []*ipld.EthTx
txTrieNodes []*ipld.EthTxTrie
logTrieNodes [][]node.Node
logLeafNodeCIDs [][]cid.Cid
rctLeafNodeCIDs []cid.Cid
}
// processReceiptsAndTxs publishes and indexes receipt and transaction IPLDs in Postgres
func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *sqlx.Tx, args processArgs) error {
// Process receipts and txs
signer := types.MakeSigner(sdi.chainConfig, args.blockNumber)
for i, receipt := range args.receipts {
// tx that corresponds with this receipt
trx := args.txs[i]
from, err := types.Sender(signer, trx)
if err != nil {
return fmt.Errorf("error deriving tx sender: %v", err)
}
for _, trie := range args.logTrieNodes[i] {
if err = shared.PublishIPLD(tx, trie); err != nil {
return fmt.Errorf("error publishing log trie node IPLD: %w", err)
}
}
// publish the txs and receipts
txNode := args.txNodes[i]
if err := shared.PublishIPLD(tx, txNode); err != nil {
return fmt.Errorf("error publishing tx IPLD: %v", err)
}
// Indexing
// extract topic and contract data from the receipt for indexing
mappedContracts := make(map[string]bool) // use map to avoid duplicate addresses
logDataSet := make([]*models.LogsModel, len(receipt.Logs))
for idx, l := range receipt.Logs {
topicSet := make([]string, 4)
for ti, topic := range l.Topics {
topicSet[ti] = topic.Hex()
}
if !args.logLeafNodeCIDs[i][idx].Defined() {
return fmt.Errorf("invalid log cid")
}
mappedContracts[l.Address.String()] = true
logDataSet[idx] = &models.LogsModel{
ID: 0,
Address: l.Address.String(),
Index: int64(l.Index),
Data: l.Data,
LeafCID: args.logLeafNodeCIDs[i][idx].String(),
LeafMhKey: shared.MultihashKeyFromCID(args.logLeafNodeCIDs[i][idx]),
Topic0: topicSet[0],
Topic1: topicSet[1],
Topic2: topicSet[2],
Topic3: topicSet[3],
}
}
// these are the contracts seen in the logs
logContracts := make([]string, 0, len(mappedContracts))
for addr := range mappedContracts {
logContracts = append(logContracts, addr)
}
// this is the contract address if this receipt is for a contract creation tx
contract := shared.HandleZeroAddr(receipt.ContractAddress)
var contractHash string
if contract != "" {
contractHash = crypto.Keccak256Hash(common.HexToAddress(contract).Bytes()).String()
}
// index tx first so that the receipt can reference it by FK
txModel := models.TxModel{
Dst: shared.HandleZeroAddrPointer(trx.To()),
Src: shared.HandleZeroAddr(from),
TxHash: trx.Hash().String(),
Index: int64(i),
Data: trx.Data(),
CID: txNode.Cid().String(),
MhKey: shared.MultihashKeyFromCID(txNode.Cid()),
}
txType := trx.Type()
if txType != types.LegacyTxType {
txModel.Type = &txType
}
txID, err := sdi.dbWriter.upsertTransactionCID(tx, txModel, args.headerID)
if err != nil {
return err
}
// index access list if this is one
for j, accessListElement := range trx.AccessList() {
storageKeys := make([]string, len(accessListElement.StorageKeys))
for k, storageKey := range accessListElement.StorageKeys {
storageKeys[k] = storageKey.Hex()
}
accessListElementModel := models.AccessListElementModel{
Index: int64(j),
Address: accessListElement.Address.Hex(),
StorageKeys: storageKeys,
}
if err := sdi.dbWriter.upsertAccessListElement(tx, accessListElementModel, txID); err != nil {
return err
}
}
// index the receipt
if !args.rctLeafNodeCIDs[i].Defined() {
return fmt.Errorf("invalid receipt leaf node cid")
}
rctModel := &models.ReceiptModel{
Contract: contract,
ContractHash: contractHash,
LeafCID: args.rctLeafNodeCIDs[i].String(),
LeafMhKey: shared.MultihashKeyFromCID(args.rctLeafNodeCIDs[i]),
LogRoot: args.rctNodes[i].LogRoot.String(),
}
if len(receipt.PostState) == 0 {
rctModel.PostStatus = receipt.Status
} else {
rctModel.PostState = common.Bytes2Hex(receipt.PostState)
}
receiptID, err := sdi.dbWriter.upsertReceiptCID(tx, rctModel, txID)
if err != nil {
return err
}
if err = sdi.dbWriter.upsertLogCID(tx, logDataSet, receiptID); err != nil {
return err
}
}
// publish trie nodes, these aren't indexed directly
for _, n := range args.txTrieNodes {
if err := shared.PublishIPLD(tx, n); err != nil {
return fmt.Errorf("error publishing tx trie node IPLD: %w", err)
}
}
for _, n := range args.rctTrieNodes {
if err := shared.PublishIPLD(tx, n); err != nil {
return fmt.Errorf("error publishing rct trie node IPLD: %w", err)
}
}
return nil
}
// PushStateNode publishes and indexes a state diff node object (including any child storage nodes) in the IPLD database
func (sdi *StateDiffIndexer) PushStateNode(tx *BlockTx, stateNode sdtypes.StateNode) error {
// publish the state node
if stateNode.NodeType == sdtypes.Removed {
// short circuit if it is a Removed node
// this assumes the db has been initialized and a public.blocks entry for the Removed node is present
stateModel := models.StateNodeModel{
Path: stateNode.Path,
StateKey: common.BytesToHash(stateNode.LeafKey).String(),
CID: RemovedNodeStateCID,
MhKey: RemovedNodeMhKey,
NodeType: stateNode.NodeType.Int(),
}
_, err := sdi.dbWriter.upsertStateCID(tx.dbtx, stateModel, tx.headerID)
return err
}
stateCIDStr, stateMhKey, err := shared.PublishRaw(tx.dbtx, ipld.MEthStateTrie, multihash.KECCAK_256, stateNode.NodeValue)
if err != nil {
return fmt.Errorf("error publishing state node IPLD: %v", err)
}
stateModel := models.StateNodeModel{
Path: stateNode.Path,
StateKey: common.BytesToHash(stateNode.LeafKey).String(),
CID: stateCIDStr,
MhKey: stateMhKey,
NodeType: stateNode.NodeType.Int(),
}
// index the state node, collect the stateID to reference by FK
stateID, err := sdi.dbWriter.upsertStateCID(tx.dbtx, stateModel, tx.headerID)
if err != nil {
return err
}
// if we have a leaf, decode and index the account data
if stateNode.NodeType == sdtypes.Leaf {
var i []interface{}
if err := rlp.DecodeBytes(stateNode.NodeValue, &i); err != nil {
return fmt.Errorf("error decoding state leaf node rlp: %s", err.Error())
}
if len(i) != 2 {
return fmt.Errorf("eth IPLDPublisher expected state leaf node rlp to decode into two elements")
}
var account types.StateAccount
if err := rlp.DecodeBytes(i[1].([]byte), &account); err != nil {
return fmt.Errorf("error decoding state account rlp: %s", err.Error())
}
accountModel := models.StateAccountModel{
Balance: account.Balance.String(),
Nonce: account.Nonce,
CodeHash: account.CodeHash,
StorageRoot: account.Root.String(),
}
if err := sdi.dbWriter.upsertStateAccount(tx.dbtx, accountModel, stateID); err != nil {
return err
}
}
// if there are any storage nodes associated with this node, publish and index them
for _, storageNode := range stateNode.StorageNodes {
if storageNode.NodeType == sdtypes.Removed {
// short circuit if it is a Removed node
// this assumes the db has been initialized and a public.blocks entry for the Removed node is present
storageModel := models.StorageNodeModel{
Path: storageNode.Path,
StorageKey: common.BytesToHash(storageNode.LeafKey).String(),
CID: RemovedNodeStorageCID,
MhKey: RemovedNodeMhKey,
NodeType: storageNode.NodeType.Int(),
}
if err := sdi.dbWriter.upsertStorageCID(tx.dbtx, storageModel, stateID); err != nil {
return err
}
continue
}
storageCIDStr, storageMhKey, err := shared.PublishRaw(tx.dbtx, ipld.MEthStorageTrie, multihash.KECCAK_256, storageNode.NodeValue)
if err != nil {
return fmt.Errorf("error publishing storage node IPLD: %v", err)
}
storageModel := models.StorageNodeModel{
Path: storageNode.Path,
StorageKey: common.BytesToHash(storageNode.LeafKey).String(),
CID: storageCIDStr,
MhKey: storageMhKey,
NodeType: storageNode.NodeType.Int(),
}
if err := sdi.dbWriter.upsertStorageCID(tx.dbtx, storageModel, stateID); err != nil {
return err
}
}
return nil
}
// PushCodeAndCodeHash publishes code and codehash pairs to the ipld database
func (sdi *StateDiffIndexer) PushCodeAndCodeHash(tx *BlockTx, codeAndCodeHash sdtypes.CodeAndCodeHash) error {
// codec doesn't matter since db key is multihash-based
mhKey, err := shared.MultihashKeyFromKeccak256(codeAndCodeHash.Hash)
if err != nil {
return fmt.Errorf("error deriving multihash key from codehash: %v", err)
}
if err := shared.PublishDirect(tx.dbtx, mhKey, codeAndCodeHash.Code); err != nil {
return fmt.Errorf("error publishing code IPLD: %v", err)
}
return nil
}

88
statediff/indexer/indexer_legacy_test.go Normal file
View File

@ -0,0 +1,88 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package indexer_test
import (
"testing"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/statediff/indexer"
"github.com/ethereum/go-ethereum/statediff/indexer/ipfs/ipld"
"github.com/ethereum/go-ethereum/statediff/indexer/mocks"
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
"github.com/ipfs/go-cid"
"github.com/multiformats/go-multihash"
"github.com/stretchr/testify/require"
)
var (
legacyData = mocks.NewLegacyData()
mockLegacyBlock *types.Block
legacyHeaderCID cid.Cid
)
func setupLegacy(t *testing.T) {
mockLegacyBlock = legacyData.MockBlock
legacyHeaderCID, _ = ipld.RawdataToCid(ipld.MEthHeader, legacyData.MockHeaderRlp, multihash.KECCAK_256)
db, err = shared.SetupDB()
require.NoError(t, err)
ind, err = indexer.NewStateDiffIndexer(legacyData.Config, db)
require.NoError(t, err)
var tx *indexer.BlockTx
tx, err = ind.PushBlock(
mockLegacyBlock,
legacyData.MockReceipts,
legacyData.MockBlock.Difficulty())
require.NoError(t, err)
defer tx.Close(err)
for _, node := range legacyData.StateDiffs {
err = ind.PushStateNode(tx, node)
require.NoError(t, err)
}
shared.ExpectEqual(t, tx.BlockNumber, legacyData.BlockNumber.Uint64())
}
func TestPublishAndIndexerLegacy(t *testing.T) {
t.Run("Publish and index header IPLDs in a legacy tx", func(t *testing.T) {
setupLegacy(t)
defer tearDown(t)
pgStr := `SELECT cid, td, reward, id, base_fee
FROM eth.header_cids
WHERE block_number = $1`
// check header was properly indexed
type res struct {
CID string
TD string
Reward string
ID int
BaseFee *int64 `db:"base_fee"`
}
header := new(res)
err = db.QueryRowx(pgStr, legacyData.BlockNumber.Uint64()).StructScan(header)
require.NoError(t, err)
shared.ExpectEqual(t, header.CID, legacyHeaderCID.String())
shared.ExpectEqual(t, header.TD, legacyData.MockBlock.Difficulty().String())
shared.ExpectEqual(t, header.Reward, "5000000000000011250")
require.Nil(t, legacyData.MockHeader.BaseFee)
require.Nil(t, header.BaseFee)
})
}

661
statediff/indexer/indexer_test.go Normal file
View File

@ -0,0 +1,661 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package indexer_test
import (
"bytes"
"fmt"
"os"
"testing"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/statediff/indexer"
"github.com/ethereum/go-ethereum/statediff/indexer/ipfs"
"github.com/ethereum/go-ethereum/statediff/indexer/ipfs/ipld"
"github.com/ethereum/go-ethereum/statediff/indexer/mocks"
"github.com/ethereum/go-ethereum/statediff/indexer/models"
"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
"github.com/ipfs/go-cid"
blockstore "github.com/ipfs/go-ipfs-blockstore"
dshelp "github.com/ipfs/go-ipfs-ds-help"
"github.com/multiformats/go-multihash"
"github.com/stretchr/testify/require"
)
var (
db *postgres.DB
err error
ind *indexer.StateDiffIndexer
ipfsPgGet = `SELECT data FROM public.blocks
WHERE key = $1`
tx1, tx2, tx3, tx4, tx5, rct1, rct2, rct3, rct4, rct5 []byte
mockBlock *types.Block
headerCID, trx1CID, trx2CID, trx3CID, trx4CID, trx5CID cid.Cid
rct1CID, rct2CID, rct3CID, rct4CID, rct5CID cid.Cid
rctLeaf1, rctLeaf2, rctLeaf3, rctLeaf4, rctLeaf5 []byte
state1CID, state2CID, storageCID cid.Cid
)
func expectTrue(t *testing.T, value bool) {
if !value {
t.Fatalf("Assertion failed")
}
}
func init() {
if os.Getenv("MODE") != "statediff" {
fmt.Println("Skipping statediff test")
os.Exit(0)
}
mockBlock = mocks.MockBlock
txs, rcts := mocks.MockBlock.Transactions(), mocks.MockReceipts
buf := new(bytes.Buffer)
txs.EncodeIndex(0, buf)
tx1 = make([]byte, buf.Len())
copy(tx1, buf.Bytes())
buf.Reset()
txs.EncodeIndex(1, buf)
tx2 = make([]byte, buf.Len())
copy(tx2, buf.Bytes())
buf.Reset()
txs.EncodeIndex(2, buf)
tx3 = make([]byte, buf.Len())
copy(tx3, buf.Bytes())
buf.Reset()
txs.EncodeIndex(3, buf)
tx4 = make([]byte, buf.Len())
copy(tx4, buf.Bytes())
buf.Reset()
txs.EncodeIndex(4, buf)
tx5 = make([]byte, buf.Len())
copy(tx5, buf.Bytes())
buf.Reset()
rcts.EncodeIndex(0, buf)
rct1 = make([]byte, buf.Len())
copy(rct1, buf.Bytes())
buf.Reset()
rcts.EncodeIndex(1, buf)
rct2 = make([]byte, buf.Len())
copy(rct2, buf.Bytes())
buf.Reset()
rcts.EncodeIndex(2, buf)
rct3 = make([]byte, buf.Len())
copy(rct3, buf.Bytes())
buf.Reset()
rcts.EncodeIndex(3, buf)
rct4 = make([]byte, buf.Len())
copy(rct4, buf.Bytes())
buf.Reset()
rcts.EncodeIndex(4, buf)
rct5 = make([]byte, buf.Len())
copy(rct5, buf.Bytes())
buf.Reset()
headerCID, _ = ipld.RawdataToCid(ipld.MEthHeader, mocks.MockHeaderRlp, multihash.KECCAK_256)
trx1CID, _ = ipld.RawdataToCid(ipld.MEthTx, tx1, multihash.KECCAK_256)
trx2CID, _ = ipld.RawdataToCid(ipld.MEthTx, tx2, multihash.KECCAK_256)
trx3CID, _ = ipld.RawdataToCid(ipld.MEthTx, tx3, multihash.KECCAK_256)
trx4CID, _ = ipld.RawdataToCid(ipld.MEthTx, tx4, multihash.KECCAK_256)
trx5CID, _ = ipld.RawdataToCid(ipld.MEthTx, tx5, multihash.KECCAK_256)
/*
rct1Node, _ := ipld.NewReceipt(rcts[0])
rct2Node, _ := ipld.NewReceipt(rcts[1])
rct3Node, _ := ipld.NewReceipt(rcts[2])
rct4Node, _ := ipld.NewReceipt(rcts[3])
rct5Node, _ := ipld.NewReceipt(rcts[4])
*/
state1CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, mocks.ContractLeafNode, multihash.KECCAK_256)
state2CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, mocks.AccountLeafNode, multihash.KECCAK_256)
storageCID, _ = ipld.RawdataToCid(ipld.MEthStorageTrie, mocks.StorageLeafNode, multihash.KECCAK_256)
receiptTrie := ipld.NewRctTrie()
receiptTrie.Add(0, rct1)
receiptTrie.Add(1, rct2)
receiptTrie.Add(2, rct3)
receiptTrie.Add(3, rct4)
receiptTrie.Add(4, rct5)
rctLeafNodes, keys, _ := receiptTrie.GetLeafNodes()
rctleafNodeCids := make([]cid.Cid, len(rctLeafNodes))
orderedRctLeafNodes := make([][]byte, len(rctLeafNodes))
for i, rln := range rctLeafNodes {
var idx uint
r := bytes.NewReader(keys[i].TrieKey)
rlp.Decode(r, &idx)
rctleafNodeCids[idx] = rln.Cid()
orderedRctLeafNodes[idx] = rln.RawData()
}
rct1CID = rctleafNodeCids[0]
rct2CID = rctleafNodeCids[1]
rct3CID = rctleafNodeCids[2]
rct4CID = rctleafNodeCids[3]
rct5CID = rctleafNodeCids[4]
rctLeaf1 = orderedRctLeafNodes[0]
rctLeaf2 = orderedRctLeafNodes[1]
rctLeaf3 = orderedRctLeafNodes[2]
rctLeaf4 = orderedRctLeafNodes[3]
rctLeaf5 = orderedRctLeafNodes[4]
}
func setup(t *testing.T) {
db, err = shared.SetupDB()
if err != nil {
t.Fatal(err)
}
ind, err = indexer.NewStateDiffIndexer(mocks.TestConfig, db)
require.NoError(t, err)
var tx *indexer.BlockTx
tx, err = ind.PushBlock(
mockBlock,
mocks.MockReceipts,
mocks.MockBlock.Difficulty())
if err != nil {
t.Fatal(err)
}
defer tx.Close(err)
for _, node := range mocks.StateDiffs {
err = ind.PushStateNode(tx, node)
if err != nil {
t.Fatal(err)
}
}
shared.ExpectEqual(t, tx.BlockNumber, mocks.BlockNumber.Uint64())
}
func tearDown(t *testing.T) {
indexer.TearDownDB(t, db)
}
func TestPublishAndIndexer(t *testing.T) {
t.Run("Publish and index header IPLDs in a single tx", func(t *testing.T) {
setup(t)
defer tearDown(t)
pgStr := `SELECT cid, td, reward, id, base_fee
FROM eth.header_cids
WHERE block_number = $1`
// check header was properly indexed
type res struct {
CID string
TD string
Reward string
ID int
BaseFee *int64 `db:"base_fee"`
}
header := new(res)
err = db.QueryRowx(pgStr, mocks.BlockNumber.Uint64()).StructScan(header)
if err != nil {
t.Fatal(err)
}
shared.ExpectEqual(t, header.CID, headerCID.String())
shared.ExpectEqual(t, header.TD, mocks.MockBlock.Difficulty().String())
shared.ExpectEqual(t, header.Reward, "2000000000000021250")
shared.ExpectEqual(t, *header.BaseFee, mocks.MockHeader.BaseFee.Int64())
dc, err := cid.Decode(header.CID)
if err != nil {
t.Fatal(err)
}
mhKey := dshelp.MultihashToDsKey(dc.Hash())
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
var data []byte
err = db.Get(&data, ipfsPgGet, prefixedKey)
if err != nil {
t.Fatal(err)
}
shared.ExpectEqual(t, data, mocks.MockHeaderRlp)
})
t.Run("Publish and index transaction IPLDs in a single tx", func(t *testing.T) {
setup(t)
defer tearDown(t)
// check that txs were properly indexed
trxs := make([]string, 0)
pgStr := `SELECT transaction_cids.cid FROM eth.transaction_cids INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id)
WHERE header_cids.block_number = $1`
err = db.Select(&trxs, pgStr, mocks.BlockNumber.Uint64())
if err != nil {
t.Fatal(err)
}
shared.ExpectEqual(t, len(trxs), 5)
expectTrue(t, shared.ListContainsString(trxs, trx1CID.String()))
expectTrue(t, shared.ListContainsString(trxs, trx2CID.String()))
expectTrue(t, shared.ListContainsString(trxs, trx3CID.String()))
expectTrue(t, shared.ListContainsString(trxs, trx4CID.String()))
expectTrue(t, shared.ListContainsString(trxs, trx5CID.String()))
// and published
for _, c := range trxs {
dc, err := cid.Decode(c)
if err != nil {
t.Fatal(err)
}
mhKey := dshelp.MultihashToDsKey(dc.Hash())
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
var data []byte
err = db.Get(&data, ipfsPgGet, prefixedKey)
if err != nil {
t.Fatal(err)
}
switch c {
case trx1CID.String():
shared.ExpectEqual(t, data, tx1)
var txType *uint8
pgStr = `SELECT tx_type FROM eth.transaction_cids WHERE cid = $1`
err = db.Get(&txType, pgStr, c)
if err != nil {
t.Fatal(err)
}
if txType != nil {
t.Fatalf("expected nil tx_type, got %d", *txType)
}
case trx2CID.String():
shared.ExpectEqual(t, data, tx2)
var txType *uint8
pgStr = `SELECT tx_type FROM eth.transaction_cids WHERE cid = $1`
err = db.Get(&txType, pgStr, c)
if err != nil {
t.Fatal(err)
}
if txType != nil {
t.Fatalf("expected nil tx_type, got %d", *txType)
}
case trx3CID.String():
shared.ExpectEqual(t, data, tx3)
var txType *uint8
pgStr = `SELECT tx_type FROM eth.transaction_cids WHERE cid = $1`
err = db.Get(&txType, pgStr, c)
if err != nil {
t.Fatal(err)
}
if txType != nil {
t.Fatalf("expected nil tx_type, got %d", *txType)
}
case trx4CID.String():
shared.ExpectEqual(t, data, tx4)
var txType *uint8
pgStr = `SELECT tx_type FROM eth.transaction_cids WHERE cid = $1`
err = db.Get(&txType, pgStr, c)
if err != nil {
t.Fatal(err)
}
if *txType != types.AccessListTxType {
t.Fatalf("expected AccessListTxType (1), got %d", *txType)
}
accessListElementModels := make([]models.AccessListElementModel, 0)
pgStr = `SELECT access_list_element.* FROM eth.access_list_element INNER JOIN eth.transaction_cids ON (tx_id = transaction_cids.id) WHERE cid = $1 ORDER BY access_list_element.index ASC`
err = db.Select(&accessListElementModels, pgStr, c)
if err != nil {
t.Fatal(err)
}
if len(accessListElementModels) != 2 {
t.Fatalf("expected two access list entries, got %d", len(accessListElementModels))
}
model1 := models.AccessListElementModel{
Index: accessListElementModels[0].Index,
Address: accessListElementModels[0].Address,
}
model2 := models.AccessListElementModel{
Index: accessListElementModels[1].Index,
Address: accessListElementModels[1].Address,
StorageKeys: accessListElementModels[1].StorageKeys,
}
shared.ExpectEqual(t, model1, mocks.AccessListEntry1Model)
shared.ExpectEqual(t, model2, mocks.AccessListEntry2Model)
case trx5CID.String():
shared.ExpectEqual(t, data, tx5)
var txType *uint8
pgStr = `SELECT tx_type FROM eth.transaction_cids WHERE cid = $1`
err = db.Get(&txType, pgStr, c)
if err != nil {
t.Fatal(err)
}
if *txType != types.DynamicFeeTxType {
t.Fatalf("expected DynamicFeeTxType (2), got %d", *txType)
}
}
}
})
t.Run("Publish and index log IPLDs for multiple receipt of a specific block", func(t *testing.T) {
setup(t)
defer tearDown(t)
rcts := make([]string, 0)
pgStr := `SELECT receipt_cids.leaf_cid FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
WHERE receipt_cids.tx_id = transaction_cids.id
AND transaction_cids.header_id = header_cids.id
AND header_cids.block_number = $1
ORDER BY transaction_cids.index`
err = db.Select(&rcts, pgStr, mocks.BlockNumber.Uint64())
if err != nil {
t.Fatal(err)
}
type logIPLD struct {
Index int `db:"index"`
Address string `db:"address"`
Data []byte `db:"data"`
Topic0 string `db:"topic0"`
Topic1 string `db:"topic1"`
}
for i := range rcts {
results := make([]logIPLD, 0)
pgStr = `SELECT log_cids.index, log_cids.address, log_cids.Topic0, log_cids.Topic1, data FROM eth.log_cids
INNER JOIN eth.receipt_cids ON (log_cids.receipt_id = receipt_cids.id)
INNER JOIN public.blocks ON (log_cids.leaf_mh_key = blocks.key)
WHERE receipt_cids.leaf_cid = $1 ORDER BY eth.log_cids.index ASC`
err = db.Select(&results, pgStr, rcts[i])
require.NoError(t, err)
// expecting MockLog1 and MockLog2 for mockReceipt4
expectedLogs := mocks.MockReceipts[i].Logs
shared.ExpectEqual(t, len(results), len(expectedLogs))
var nodeElements []interface{}
for idx, r := range results {
// Decode the log leaf node.
err = rlp.DecodeBytes(r.Data, &nodeElements)
require.NoError(t, err)
logRaw, err := rlp.EncodeToBytes(expectedLogs[idx])
require.NoError(t, err)
// 2nd element of the leaf node contains the encoded log data.
shared.ExpectEqual(t, logRaw, nodeElements[1].([]byte))
}
}
})
t.Run("Publish and index receipt IPLDs in a single tx", func(t *testing.T) {
setup(t)
defer tearDown(t)
// check receipts were properly indexed
rcts := make([]string, 0)
pgStr := `SELECT receipt_cids.leaf_cid FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
WHERE receipt_cids.tx_id = transaction_cids.id
AND transaction_cids.header_id = header_cids.id
AND header_cids.block_number = $1 order by transaction_cids.id`
err = db.Select(&rcts, pgStr, mocks.BlockNumber.Uint64())
if err != nil {
t.Fatal(err)
}
shared.ExpectEqual(t, len(rcts), 5)
for idx, rctLeafCID := range rcts {
result := make([]ipfs.BlockModel, 0)
pgStr = `SELECT data
FROM eth.receipt_cids
INNER JOIN public.blocks ON (receipt_cids.leaf_mh_key = public.blocks.key)
WHERE receipt_cids.leaf_cid = $1`
err = db.Select(&result, pgStr, rctLeafCID)
if err != nil {
t.Fatal(err)
}
// Decode the log leaf node.
var nodeElements []interface{}
err = rlp.DecodeBytes(result[0].Data, &nodeElements)
require.NoError(t, err)
expectedRct, err := mocks.MockReceipts[idx].MarshalBinary()
require.NoError(t, err)
shared.ExpectEqual(t, expectedRct, nodeElements[1].([]byte))
}
// and published
for _, c := range rcts {
dc, err := cid.Decode(c)
if err != nil {
t.Fatal(err)
}
mhKey := dshelp.MultihashToDsKey(dc.Hash())
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
var data []byte
err = db.Get(&data, ipfsPgGet, prefixedKey)
if err != nil {
t.Fatal(err)
}
switch c {
case rct1CID.String():
shared.ExpectEqual(t, data, rctLeaf1)
var postStatus uint64
pgStr = `SELECT post_status FROM eth.receipt_cids WHERE leaf_cid = $1`
err = db.Get(&postStatus, pgStr, c)
if err != nil {
t.Fatal(err)
}
shared.ExpectEqual(t, postStatus, mocks.ExpectedPostStatus)
case rct2CID.String():
shared.ExpectEqual(t, data, rctLeaf2)
var postState string
pgStr = `SELECT post_state FROM eth.receipt_cids WHERE leaf_cid = $1`
err = db.Get(&postState, pgStr, c)
if err != nil {
t.Fatal(err)
}
shared.ExpectEqual(t, postState, mocks.ExpectedPostState1)
case rct3CID.String():
shared.ExpectEqual(t, data, rctLeaf3)
var postState string
pgStr = `SELECT post_state FROM eth.receipt_cids WHERE leaf_cid = $1`
err = db.Get(&postState, pgStr, c)
if err != nil {
t.Fatal(err)
}
shared.ExpectEqual(t, postState, mocks.ExpectedPostState2)
case rct4CID.String():
shared.ExpectEqual(t, data, rctLeaf4)
var postState string
pgStr = `SELECT post_state FROM eth.receipt_cids WHERE leaf_cid = $1`
err = db.Get(&postState, pgStr, c)
if err != nil {
t.Fatal(err)
}
shared.ExpectEqual(t, postState, mocks.ExpectedPostState3)
case rct5CID.String():
shared.ExpectEqual(t, data, rctLeaf5)
var postState string
pgStr = `SELECT post_state FROM eth.receipt_cids WHERE leaf_cid = $1`
err = db.Get(&postState, pgStr, c)
if err != nil {
t.Fatal(err)
}
shared.ExpectEqual(t, postState, mocks.ExpectedPostState3)
}
}
})
t.Run("Publish and index state IPLDs in a single tx", func(t *testing.T) {
setup(t)
defer tearDown(t)
// check that state nodes were properly indexed and published
stateNodes := make([]models.StateNodeModel, 0)
pgStr := `SELECT state_cids.id, state_cids.cid, state_cids.state_leaf_key, state_cids.node_type, state_cids.state_path, state_cids.header_id
FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
WHERE header_cids.block_number = $1 AND node_type != 3`
err = db.Select(&stateNodes, pgStr, mocks.BlockNumber.Uint64())
if err != nil {
t.Fatal(err)
}
shared.ExpectEqual(t, len(stateNodes), 2)
for _, stateNode := range stateNodes {
var data []byte
dc, err := cid.Decode(stateNode.CID)
if err != nil {
t.Fatal(err)
}
mhKey := dshelp.MultihashToDsKey(dc.Hash())
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
err = db.Get(&data, ipfsPgGet, prefixedKey)
if err != nil {
t.Fatal(err)
}
pgStr = `SELECT * from eth.state_accounts WHERE state_id = $1`
var account models.StateAccountModel
err = db.Get(&account, pgStr, stateNode.ID)
if err != nil {
t.Fatal(err)
}
if stateNode.CID == state1CID.String() {
shared.ExpectEqual(t, stateNode.NodeType, 2)
shared.ExpectEqual(t, stateNode.StateKey, common.BytesToHash(mocks.ContractLeafKey).Hex())
shared.ExpectEqual(t, stateNode.Path, []byte{'\x06'})
shared.ExpectEqual(t, data, mocks.ContractLeafNode)
shared.ExpectEqual(t, account, models.StateAccountModel{
ID: account.ID,
StateID: stateNode.ID,
Balance: "0",
CodeHash: mocks.ContractCodeHash.Bytes(),
StorageRoot: mocks.ContractRoot,
Nonce: 1,
})
}
if stateNode.CID == state2CID.String() {
shared.ExpectEqual(t, stateNode.NodeType, 2)
shared.ExpectEqual(t, stateNode.StateKey, common.BytesToHash(mocks.AccountLeafKey).Hex())
shared.ExpectEqual(t, stateNode.Path, []byte{'\x0c'})
shared.ExpectEqual(t, data, mocks.AccountLeafNode)
shared.ExpectEqual(t, account, models.StateAccountModel{
ID: account.ID,
StateID: stateNode.ID,
Balance: "1000",
CodeHash: mocks.AccountCodeHash.Bytes(),
StorageRoot: mocks.AccountRoot,
Nonce: 0,
})
}
}
// check that Removed state nodes were properly indexed and published
stateNodes = make([]models.StateNodeModel, 0)
pgStr = `SELECT state_cids.id, state_cids.cid, state_cids.state_leaf_key, state_cids.node_type, state_cids.state_path, state_cids.header_id
FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
WHERE header_cids.block_number = $1 AND node_type = 3`
err = db.Select(&stateNodes, pgStr, mocks.BlockNumber.Uint64())
if err != nil {
t.Fatal(err)
}
shared.ExpectEqual(t, len(stateNodes), 1)
stateNode := stateNodes[0]
var data []byte
dc, err := cid.Decode(stateNode.CID)
if err != nil {
t.Fatal(err)
}
mhKey := dshelp.MultihashToDsKey(dc.Hash())
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
shared.ExpectEqual(t, prefixedKey, indexer.RemovedNodeMhKey)
err = db.Get(&data, ipfsPgGet, prefixedKey)
if err != nil {
t.Fatal(err)
}
shared.ExpectEqual(t, stateNode.CID, indexer.RemovedNodeStateCID)
shared.ExpectEqual(t, stateNode.Path, []byte{'\x02'})
shared.ExpectEqual(t, data, []byte{})
})
t.Run("Publish and index storage IPLDs in a single tx", func(t *testing.T) {
setup(t)
defer tearDown(t)
// check that storage nodes were properly indexed
storageNodes := make([]models.StorageNodeWithStateKeyModel, 0)
pgStr := `SELECT storage_cids.cid, state_cids.state_leaf_key, storage_cids.storage_leaf_key, storage_cids.node_type, storage_cids.storage_path
FROM eth.storage_cids, eth.state_cids, eth.header_cids
WHERE storage_cids.state_id = state_cids.id
AND state_cids.header_id = header_cids.id
AND header_cids.block_number = $1
AND storage_cids.node_type != 3`
err = db.Select(&storageNodes, pgStr, mocks.BlockNumber.Uint64())
if err != nil {
t.Fatal(err)
}
shared.ExpectEqual(t, len(storageNodes), 1)
shared.ExpectEqual(t, storageNodes[0], models.StorageNodeWithStateKeyModel{
CID: storageCID.String(),
NodeType: 2,
StorageKey: common.BytesToHash(mocks.StorageLeafKey).Hex(),
StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(),
Path: []byte{},
})
var data []byte
dc, err := cid.Decode(storageNodes[0].CID)
if err != nil {
t.Fatal(err)
}
mhKey := dshelp.MultihashToDsKey(dc.Hash())
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
err = db.Get(&data, ipfsPgGet, prefixedKey)
if err != nil {
t.Fatal(err)
}
shared.ExpectEqual(t, data, mocks.StorageLeafNode)
// check that Removed storage nodes were properly indexed
storageNodes = make([]models.StorageNodeWithStateKeyModel, 0)
pgStr = `SELECT storage_cids.cid, state_cids.state_leaf_key, storage_cids.storage_leaf_key, storage_cids.node_type, storage_cids.storage_path
FROM eth.storage_cids, eth.state_cids, eth.header_cids
WHERE storage_cids.state_id = state_cids.id
AND state_cids.header_id = header_cids.id
AND header_cids.block_number = $1
AND storage_cids.node_type = 3`
err = db.Select(&storageNodes, pgStr, mocks.BlockNumber.Uint64())
if err != nil {
t.Fatal(err)
}
shared.ExpectEqual(t, len(storageNodes), 1)
shared.ExpectEqual(t, storageNodes[0], models.StorageNodeWithStateKeyModel{
CID: indexer.RemovedNodeStorageCID,
NodeType: 3,
StorageKey: common.BytesToHash(mocks.RemovedLeafKey).Hex(),
StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(),
Path: []byte{'\x03'},
})
dc, err = cid.Decode(storageNodes[0].CID)
if err != nil {
t.Fatal(err)
}
mhKey = dshelp.MultihashToDsKey(dc.Hash())
prefixedKey = blockstore.BlockPrefix.String() + mhKey.String()
shared.ExpectEqual(t, prefixedKey, indexer.RemovedNodeMhKey)
err = db.Get(&data, ipfsPgGet, prefixedKey)
if err != nil {
t.Fatal(err)
}
shared.ExpectEqual(t, data, []byte{})
})
}

BIN
statediff/indexer/ipfs/ipld/eip2930_test_data/eth-block-12252078 Normal file
View File

Binary file not shown.

BIN
statediff/indexer/ipfs/ipld/eip2930_test_data/eth-block-12365585 Normal file
View File

Binary file not shown.

BIN
statediff/indexer/ipfs/ipld/eip2930_test_data/eth-block-12365586 Normal file
View File

Binary file not shown.

BIN
statediff/indexer/ipfs/ipld/eip2930_test_data/eth-receipts-12252078 Normal file
View File

Binary file not shown.

BIN
statediff/indexer/ipfs/ipld/eip2930_test_data/eth-receipts-12365585 Normal file
View File

Binary file not shown.

BIN
statediff/indexer/ipfs/ipld/eip2930_test_data/eth-receipts-12365586 Normal file
View File

Binary file not shown.

175
statediff/indexer/ipfs/ipld/eth_account.go Normal file
View File

@ -0,0 +1,175 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipld
import (
"encoding/json"
"fmt"
"math/big"
"github.com/ipfs/go-cid"
node "github.com/ipfs/go-ipld-format"
)
// EthAccountSnapshot (eth-account-snapshot codec 0x97)
// represents an ethereum account, i.e. a wallet address or
// a smart contract
type EthAccountSnapshot struct {
*EthAccount
cid cid.Cid
rawdata []byte
}
// EthAccount is the building block of EthAccountSnapshot.
// Or, is the former stripped of its cid and rawdata components.
type EthAccount struct {
Nonce uint64
Balance *big.Int
Root []byte // This is the storage root trie
CodeHash []byte // This is the hash of the EVM code
}
// Static (compile time) check that EthAccountSnapshot satisfies the
// node.Node interface.
var _ node.Node = (*EthAccountSnapshot)(nil)
/*
INPUT
*/
// Input should be managed by EthStateTrie
/*
OUTPUT
*/
// Output should be managed by EthStateTrie
/*
Block INTERFACE
*/
// RawData returns the binary of the RLP encode of the account snapshot.
func (as *EthAccountSnapshot) RawData() []byte {
return as.rawdata
}
// Cid returns the cid of the transaction.
func (as *EthAccountSnapshot) Cid() cid.Cid {
return as.cid
}
// String is a helper for output
func (as *EthAccountSnapshot) String() string {
return fmt.Sprintf("<EthereumAccountSnapshot %s>", as.cid)
}
// Loggable returns in a map the type of IPLD Link.
func (as *EthAccountSnapshot) Loggable() map[string]interface{} {
return map[string]interface{}{
"type": "eth-account-snapshot",
}
}
/*
Node INTERFACE
*/
// Resolve resolves a path through this node, stopping at any link boundary
// and returning the object found as well as the remaining path to traverse
func (as *EthAccountSnapshot) Resolve(p []string) (interface{}, []string, error) {
if len(p) == 0 {
return as, nil, nil
}
if len(p) > 1 {
return nil, nil, fmt.Errorf("unexpected path elements past %s", p[0])
}
switch p[0] {
case "balance":
return as.Balance, nil, nil
case "codeHash":
return &node.Link{Cid: keccak256ToCid(RawBinary, as.CodeHash)}, nil, nil
case "nonce":
return as.Nonce, nil, nil
case "root":
return &node.Link{Cid: keccak256ToCid(MEthStorageTrie, as.Root)}, nil, nil
default:
return nil, nil, ErrInvalidLink
}
}
// Tree lists all paths within the object under 'path', and up to the given depth.
// To list the entire object (similar to `find .`) pass "" and -1
func (as *EthAccountSnapshot) Tree(p string, depth int) []string {
if p != "" || depth == 0 {
return nil
}
return []string{"balance", "codeHash", "nonce", "root"}
}
// ResolveLink is a helper function that calls resolve and asserts the
// output is a link
func (as *EthAccountSnapshot) ResolveLink(p []string) (*node.Link, []string, error) {
obj, rest, err := as.Resolve(p)
if err != nil {
return nil, nil, err
}
if lnk, ok := obj.(*node.Link); ok {
return lnk, rest, nil
}
return nil, nil, fmt.Errorf("resolved item was not a link")
}
// Copy will go away. It is here to comply with the interface.
func (as *EthAccountSnapshot) Copy() node.Node {
panic("implement me")
}
// Links is a helper function that returns all links within this object
func (as *EthAccountSnapshot) Links() []*node.Link {
return nil
}
// Stat will go away. It is here to comply with the interface.
func (as *EthAccountSnapshot) Stat() (*node.NodeStat, error) {
return &node.NodeStat{}, nil
}
// Size will go away. It is here to comply with the interface.
func (as *EthAccountSnapshot) Size() (uint64, error) {
return 0, nil
}
/*
EthAccountSnapshot functions
*/
// MarshalJSON processes the transaction into readable JSON format.
func (as *EthAccountSnapshot) MarshalJSON() ([]byte, error) {
out := map[string]interface{}{
"balance": as.Balance,
"codeHash": keccak256ToCid(RawBinary, as.CodeHash),
"nonce": as.Nonce,
"root": keccak256ToCid(MEthStorageTrie, as.Root),
}
return json.Marshal(out)
}

298
statediff/indexer/ipfs/ipld/eth_account_test.go Normal file
View File

@ -0,0 +1,298 @@
package ipld
import (
"encoding/json"
"fmt"
"os"
"regexp"
"testing"
)
/*
Block INTERFACE
*/
func init() {
if os.Getenv("MODE") != "statediff" {
fmt.Println("Skipping statediff test")
os.Exit(0)
}
}
func TestAccountSnapshotBlockElements(t *testing.T) {
eas := prepareEthAccountSnapshot(t)
if fmt.Sprintf("%x", eas.RawData())[:10] != "f84e808a03" {
t.Fatal("Wrong Data")
}
if eas.Cid().String() !=
"baglqcgzasckx2alxk43cksshnztjvhfyvbbh6bkp376gtcndm5cg4fkrkhsa" {
t.Fatal("Wrong Cid")
}
}
func TestAccountSnapshotString(t *testing.T) {
eas := prepareEthAccountSnapshot(t)
if eas.String() !=
"<EthereumAccountSnapshot baglqcgzasckx2alxk43cksshnztjvhfyvbbh6bkp376gtcndm5cg4fkrkhsa>" {
t.Fatalf("Wrong String()")
}
}
func TestAccountSnapshotLoggable(t *testing.T) {
eas := prepareEthAccountSnapshot(t)
l := eas.Loggable()
if _, ok := l["type"]; !ok {
t.Fatal("Loggable map expected the field 'type'")
}
if l["type"] != "eth-account-snapshot" {
t.Fatalf("Wrong Loggable 'type' value\r\nexpected %s\r\ngot %s", "eth-account-snapshot", l["type"])
}
}
/*
Node INTERFACE
*/
func TestAccountSnapshotResolve(t *testing.T) {
eas := prepareEthAccountSnapshot(t)
// Empty path
obj, rest, err := eas.Resolve([]string{})
reas, ok := obj.(*EthAccountSnapshot)
if !ok {
t.Fatalf("Wrong type of returned object\r\nexpected %T\r\ngot %T", &EthAccountSnapshot{}, reas)
}
if reas.Cid() != eas.Cid() {
t.Fatalf("wrong returned CID\r\nexpected %s\r\ngot %s", eas.Cid().String(), reas.Cid().String())
}
if rest != nil {
t.Fatal("rest should be nil")
}
if err != nil {
t.Fatal("err should be nil")
}
// len(p) > 1
badCases := [][]string{
{"two", "elements"},
{"here", "three", "elements"},
{"and", "here", "four", "elements"},
}
for _, bc := range badCases {
obj, rest, err = eas.Resolve(bc)
if obj != nil {
t.Fatal("obj should be nil")
}
if rest != nil {
t.Fatal("rest should be nil")
}
if err.Error() != fmt.Sprintf("unexpected path elements past %s", bc[0]) {
t.Fatal("wrong error")
}
}
moreBadCases := []string{
"i",
"am",
"not",
"an",
"account",
"field",
}
for _, mbc := range moreBadCases {
obj, rest, err = eas.Resolve([]string{mbc})
if obj != nil {
t.Fatal("obj should be nil")
}
if rest != nil {
t.Fatal("rest should be nil")
}
if err != ErrInvalidLink {
t.Fatal("wrong error")
}
}
goodCases := []string{
"balance",
"codeHash",
"nonce",
"root",
}
for _, gc := range goodCases {
_, _, err = eas.Resolve([]string{gc})
if err != nil {
t.Fatalf("error should be nil %v", gc)
}
}
}
func TestAccountSnapshotTree(t *testing.T) {
eas := prepareEthAccountSnapshot(t)
// Bad cases
tree := eas.Tree("non-empty-string", 0)
if tree != nil {
t.Fatal("Expected nil to be returned")
}
tree = eas.Tree("non-empty-string", 1)
if tree != nil {
t.Fatal("Expected nil to be returned")
}
tree = eas.Tree("", 0)
if tree != nil {
t.Fatal("Expected nil to be returned")
}
// Good cases
tree = eas.Tree("", 1)
lookupElements := map[string]interface{}{
"balance": nil,
"codeHash": nil,
"nonce": nil,
"root": nil,
}
if len(tree) != len(lookupElements) {
t.Fatalf("Wrong number of elements\r\nexpected %d\r\ngot %d", len(lookupElements), len(tree))
}
for _, te := range tree {
if _, ok := lookupElements[te]; !ok {
t.Fatalf("Unexpected Element: %v", te)
}
}
}
func TestAccountSnapshotResolveLink(t *testing.T) {
eas := prepareEthAccountSnapshot(t)
// bad case
obj, rest, err := eas.ResolveLink([]string{"supercalifragilist"})
if obj != nil {
t.Fatalf("Expected obj to be nil")
}
if rest != nil {
t.Fatal("Expected rest to be nil")
}
if err != ErrInvalidLink {
t.Fatal("Wrong error")
}
// good case
obj, rest, err = eas.ResolveLink([]string{"nonce"})
if obj != nil {
t.Fatalf("Expected obj to be nil")
}
if rest != nil {
t.Fatal("Expected rest to be nil")
}
if err.Error() != "resolved item was not a link" {
t.Fatal("Wrong error")
}
}
func TestAccountSnapshotCopy(t *testing.T) {
eas := prepareEthAccountSnapshot(t)
defer func() {
r := recover()
if r == nil {
t.Fatal("Expected panic")
}
if r != "implement me" {
t.Fatalf("Wrong panic message\r\n expected %s\r\ngot %s", "'implement me'", r)
}
}()
_ = eas.Copy()
}
func TestAccountSnapshotLinks(t *testing.T) {
eas := prepareEthAccountSnapshot(t)
if eas.Links() != nil {
t.Fatal("Links() expected to return nil")
}
}
func TestAccountSnapshotStat(t *testing.T) {
eas := prepareEthAccountSnapshot(t)
obj, err := eas.Stat()
if obj == nil {
t.Fatal("Expected a not null object node.NodeStat")
}
if err != nil {
t.Fatal("Expected a nil error")
}
}
func TestAccountSnapshotSize(t *testing.T) {
eas := prepareEthAccountSnapshot(t)
size, err := eas.Size()
if size != uint64(0) {
t.Fatalf("Wrong size\r\nexpected %d\r\ngot %d", 0, size)
}
if err != nil {
t.Fatal("Expected a nil error")
}
}
/*
EthAccountSnapshot functions
*/
func TestAccountSnapshotMarshalJSON(t *testing.T) {
eas := prepareEthAccountSnapshot(t)
jsonOutput, err := eas.MarshalJSON()
checkError(err, t)
var data map[string]interface{}
err = json.Unmarshal(jsonOutput, &data)
checkError(err, t)
balanceExpression := regexp.MustCompile(`{"balance":16011846000000000000000,`)
if !balanceExpression.MatchString(string(jsonOutput)) {
t.Fatal("Balance expression not found")
}
code, _ := data["codeHash"].(map[string]interface{})
if fmt.Sprintf("%s", code["/"]) !=
"bafkrwigf2jdadbxxem6je7t5wlomoa6a4ualmu6kqittw6723acf3bneoa" {
t.Fatalf("Wrong Marshaled Value\r\nexpected %s\r\ngot %s", "bafkrwigf2jdadbxxem6je7t5wlomoa6a4ualmu6kqittw6723acf3bneoa", fmt.Sprintf("%s", code["/"]))
}
if fmt.Sprintf("%v", data["nonce"]) != "0" {
t.Fatalf("Wrong Marshaled Value\r\nexpected %s\r\ngot %s", "0", fmt.Sprintf("%v", data["nonce"]))
}
root, _ := data["root"].(map[string]interface{})
if fmt.Sprintf("%s", root["/"]) !=
"bagmacgzak3ub6fy3zrk2n74dixtjfqhynznurya3tfwk3qabmix3ly3dwqqq" {
t.Fatalf("Wrong Marshaled Value\r\nexpected %s\r\ngot %s", "bagmacgzak3ub6fy3zrk2n74dixtjfqhynznurya3tfwk3qabmix3ly3dwqqq", fmt.Sprintf("%s", root["/"]))
}
}
/*
AUXILIARS
*/
func prepareEthAccountSnapshot(t *testing.T) *EthAccountSnapshot {
fi, err := os.Open("test_data/eth-state-trie-rlp-c9070d")
checkError(err, t)
output, err := FromStateTrieRLPFile(fi)
checkError(err, t)
return output.elements[1].(*EthAccountSnapshot)
}

293
statediff/indexer/ipfs/ipld/eth_header.go Normal file
View File

@ -0,0 +1,293 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipld
import (
"encoding/json"
"fmt"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ipfs/go-cid"
node "github.com/ipfs/go-ipld-format"
mh "github.com/multiformats/go-multihash"
)
// EthHeader (eth-block, codec 0x90), represents an ethereum block header
type EthHeader struct {
*types.Header
cid cid.Cid
rawdata []byte
}
// Static (compile time) check that EthHeader satisfies the node.Node interface.
var _ node.Node = (*EthHeader)(nil)
/*
INPUT
*/
// NewEthHeader converts a *types.Header into an EthHeader IPLD node
func NewEthHeader(header *types.Header) (*EthHeader, error) {
headerRLP, err := rlp.EncodeToBytes(header)
if err != nil {
return nil, err
}
c, err := RawdataToCid(MEthHeader, headerRLP, mh.KECCAK_256)
if err != nil {
return nil, err
}
return &EthHeader{
Header: header,
cid: c,
rawdata: headerRLP,
}, nil
}
/*
OUTPUT
*/
// DecodeEthHeader takes a cid and its raw binary data
// from IPFS and returns an EthTx object for further processing.
func DecodeEthHeader(c cid.Cid, b []byte) (*EthHeader, error) {
h := new(types.Header)
if err := rlp.DecodeBytes(b, h); err != nil {
return nil, err
}
return &EthHeader{
Header: h,
cid: c,
rawdata: b,
}, nil
}
/*
Block INTERFACE
*/
// RawData returns the binary of the RLP encode of the block header.
func (b *EthHeader) RawData() []byte {
return b.rawdata
}
// Cid returns the cid of the block header.
func (b *EthHeader) Cid() cid.Cid {
return b.cid
}
// String is a helper for output
func (b *EthHeader) String() string {
return fmt.Sprintf("<EthHeader %s>", b.cid)
}
// Loggable returns a map the type of IPLD Link.
func (b *EthHeader) Loggable() map[string]interface{} {
return map[string]interface{}{
"type": "eth-header",
}
}
/*
Node INTERFACE
*/
// Resolve resolves a path through this node, stopping at any link boundary
// and returning the object found as well as the remaining path to traverse
func (b *EthHeader) Resolve(p []string) (interface{}, []string, error) {
if len(p) == 0 {
return b, nil, nil
}
first, rest := p[0], p[1:]
switch first {
case "parent":
return &node.Link{Cid: commonHashToCid(MEthHeader, b.ParentHash)}, rest, nil
case "receipts":
return &node.Link{Cid: commonHashToCid(MEthTxReceiptTrie, b.ReceiptHash)}, rest, nil
case "root":
return &node.Link{Cid: commonHashToCid(MEthStateTrie, b.Root)}, rest, nil
case "tx":
return &node.Link{Cid: commonHashToCid(MEthTxTrie, b.TxHash)}, rest, nil
case "uncles":
return &node.Link{Cid: commonHashToCid(MEthHeaderList, b.UncleHash)}, rest, nil
}
if len(p) != 1 {
return nil, nil, fmt.Errorf("unexpected path elements past %s", first)
}
switch first {
case "bloom":
return b.Bloom, nil, nil
case "coinbase":
return b.Coinbase, nil, nil
case "difficulty":
return b.Difficulty, nil, nil
case "extra":
// This is a []byte. By default they are marshalled into Base64.
return fmt.Sprintf("0x%x", b.Extra), nil, nil
case "gaslimit":
return b.GasLimit, nil, nil
case "gasused":
return b.GasUsed, nil, nil
case "mixdigest":
return b.MixDigest, nil, nil
case "nonce":
return b.Nonce, nil, nil
case "number":
return b.Number, nil, nil
case "time":
return b.Time, nil, nil
default:
return nil, nil, ErrInvalidLink
}
}
// Tree lists all paths within the object under 'path', and up to the given depth.
// To list the entire object (similar to `find .`) pass "" and -1
func (b *EthHeader) Tree(p string, depth int) []string {
if p != "" || depth == 0 {
return nil
}
return []string{
"time",
"bloom",
"coinbase",
"difficulty",
"extra",
"gaslimit",
"gasused",
"mixdigest",
"nonce",
"number",
"parent",
"receipts",
"root",
"tx",
"uncles",
}
}
// ResolveLink is a helper function that allows easier traversal of links through blocks
func (b *EthHeader) ResolveLink(p []string) (*node.Link, []string, error) {
obj, rest, err := b.Resolve(p)
if err != nil {
return nil, nil, err
}
if lnk, ok := obj.(*node.Link); ok {
return lnk, rest, nil
}
return nil, nil, fmt.Errorf("resolved item was not a link")
}
// Copy will go away. It is here to comply with the Node interface.
func (b *EthHeader) Copy() node.Node {
panic("implement me")
}
// Links is a helper function that returns all links within this object
// HINT: Use `ipfs refs <cid>`
func (b *EthHeader) Links() []*node.Link {
return []*node.Link{
{Cid: commonHashToCid(MEthHeader, b.ParentHash)},
{Cid: commonHashToCid(MEthTxReceiptTrie, b.ReceiptHash)},
{Cid: commonHashToCid(MEthStateTrie, b.Root)},
{Cid: commonHashToCid(MEthTxTrie, b.TxHash)},
{Cid: commonHashToCid(MEthHeaderList, b.UncleHash)},
}
}
// Stat will go away. It is here to comply with the Node interface.
func (b *EthHeader) Stat() (*node.NodeStat, error) {
return &node.NodeStat{}, nil
}
// Size will go away. It is here to comply with the Node interface.
func (b *EthHeader) Size() (uint64, error) {
return 0, nil
}
/*
EthHeader functions
*/
// MarshalJSON processes the block header into readable JSON format,
// converting the right links into their cids, and keeping the original
// hex hash, allowing the user to simplify external queries.
func (b *EthHeader) MarshalJSON() ([]byte, error) {
out := map[string]interface{}{
"time": b.Time,
"bloom": b.Bloom,
"coinbase": b.Coinbase,
"difficulty": b.Difficulty,
"extra": fmt.Sprintf("0x%x", b.Extra),
"gaslimit": b.GasLimit,
"gasused": b.GasUsed,
"mixdigest": b.MixDigest,
"nonce": b.Nonce,
"number": b.Number,
"parent": commonHashToCid(MEthHeader, b.ParentHash),
"receipts": commonHashToCid(MEthTxReceiptTrie, b.ReceiptHash),
"root": commonHashToCid(MEthStateTrie, b.Root),
"tx": commonHashToCid(MEthTxTrie, b.TxHash),
"uncles": commonHashToCid(MEthHeaderList, b.UncleHash),
}
return json.Marshal(out)
}
// objJSONHeader defines the output of the JSON RPC API for either
// "eth_BlockByHash" or "eth_BlockByHeader".
type objJSONHeader struct {
Result objJSONHeaderResult `json:"result"`
}
// objJSONBLockResult is the nested struct that takes
// the contents of the JSON field "result".
type objJSONHeaderResult struct {
types.Header // Use its fields and unmarshaler
*objJSONHeaderResultExt // Add these fields to the parsing
}
// objJSONBLockResultExt facilitates the composition
// of the field "result", adding to the
// `types.Header` fields, both ommers (their hashes) and transactions.
type objJSONHeaderResultExt struct {
OmmerHashes []common.Hash `json:"uncles"`
Transactions []*types.Transaction `json:"transactions"`
}
// UnmarshalJSON overrides the function types.Header.UnmarshalJSON, allowing us
// to parse the fields of Header, plus ommer hashes and transactions.
// (yes, ommer hashes. You will need to "eth_getUncleCountByBlockHash" per each ommer)
func (o *objJSONHeaderResult) UnmarshalJSON(input []byte) error {
err := o.Header.UnmarshalJSON(input)
if err != nil {
return err
}
o.objJSONHeaderResultExt = &objJSONHeaderResultExt{}
err = json.Unmarshal(input, o.objJSONHeaderResultExt)
return err
}

585
statediff/indexer/ipfs/ipld/eth_header_test.go Normal file
View File

@ -0,0 +1,585 @@
package ipld
import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"runtime"
"strconv"
"testing"
"github.com/ethereum/go-ethereum/core/types"
block "github.com/ipfs/go-block-format"
node "github.com/ipfs/go-ipld-format"
"github.com/multiformats/go-multihash"
)
func TestBlockBodyRlpParsing(t *testing.T) {
fi, err := os.Open("test_data/eth-block-body-rlp-999999")
checkError(err, t)
output, _, _, err := FromBlockRLP(fi)
checkError(err, t)
testEthBlockFields(output, t)
}
func TestBlockHeaderRlpParsing(t *testing.T) {
fi, err := os.Open("test_data/eth-block-header-rlp-999999")
checkError(err, t)
output, _, _, err := FromBlockRLP(fi)
checkError(err, t)
testEthBlockFields(output, t)
}
func TestBlockBodyJsonParsing(t *testing.T) {
fi, err := os.Open("test_data/eth-block-body-json-999999")
checkError(err, t)
output, _, _, err := FromBlockJSON(fi)
checkError(err, t)
testEthBlockFields(output, t)
}
func TestEthBlockProcessTransactionsError(t *testing.T) {
// Let's just change one byte in a field of one of these transactions.
fi, err := os.Open("test_data/error-tx-eth-block-body-json-999999")
checkError(err, t)
_, _, _, err = FromBlockJSON(fi)
if err == nil {
t.Fatal("Expected an error")
}
}
// TestDecodeBlockHeader should work for both inputs (block header and block body)
// as what we are storing is just the block header
func TestDecodeBlockHeader(t *testing.T) {
storedEthBlock := prepareStoredEthBlock("test_data/eth-block-header-rlp-999999", t)
ethBlock, err := DecodeEthHeader(storedEthBlock.Cid(), storedEthBlock.RawData())
checkError(err, t)
testEthBlockFields(ethBlock, t)
}
func TestEthBlockString(t *testing.T) {
ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t)
if ethBlock.String() != "<EthHeader bagiacgzawt5236hkiuvrhfyy4jya3qitlt6icfcqgheew6vsptlraokppm4a>" {
t.Fatalf("Wrong String()\r\nexpected %s\r\ngot %s", "<EthHeader bagiacgzawt5236hkiuvrhfyy4jya3qitlt6icfcqgheew6vsptlraokppm4a>", ethBlock.String())
}
}
func TestEthBlockLoggable(t *testing.T) {
ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t)
l := ethBlock.Loggable()
if _, ok := l["type"]; !ok {
t.Fatal("Loggable map expected the field 'type'")
}
if l["type"] != "eth-header" {
t.Fatalf("Wrong Loggable 'type' value\r\nexpected %s\r\ngot %s", "eth-header", l["type"])
}
}
func TestEthBlockJSONMarshal(t *testing.T) {
ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t)
jsonOutput, err := ethBlock.MarshalJSON()
checkError(err, t)
var data map[string]interface{}
err = json.Unmarshal(jsonOutput, &data)
checkError(err, t)
// Testing all fields is boring, but can help us to avoid
// that dreaded regression
if data["bloom"].(string)[:10] != "0x00000000" {
t.Fatalf("Wrong Bloom\r\nexpected %s\r\ngot %s", "0x00000000", data["bloom"].(string)[:10])
t.Fatal("Wrong Bloom")
}
if data["coinbase"] != "0x52bc44d5378309ee2abf1539bf71de1b7d7be3b5" {
t.Fatalf("Wrong coinbase\r\nexpected %s\r\ngot %s", "0x52bc44d5378309ee2abf1539bf71de1b7d7be3b5", data["coinbase"])
}
if parseFloat(data["difficulty"]) != "12555463106190" {
t.Fatalf("Wrong Difficulty\r\nexpected %s\r\ngot %s", "12555463106190", parseFloat(data["difficulty"]))
}
if data["extra"] != "0xd783010303844765746887676f312e342e32856c696e7578" {
t.Fatalf("Wrong Extra\r\nexpected %s\r\ngot %s", "0xd783010303844765746887676f312e342e32856c696e7578", data["extra"])
}
if parseFloat(data["gaslimit"]) != "3141592" {
t.Fatalf("Wrong Gas limit\r\nexpected %s\r\ngot %s", "3141592", parseFloat(data["gaslimit"]))
}
if parseFloat(data["gasused"]) != "231000" {
t.Fatalf("Wrong Gas used\r\nexpected %s\r\ngot %s", "231000", parseFloat(data["gasused"]))
}
if data["mixdigest"] != "0x5b10f4a08a6c209d426f6158bd24b574f4f7b7aa0099c67c14a1f693b4dd04d0" {
t.Fatalf("Wrong Mix digest\r\nexpected %s\r\ngot %s", "0x5b10f4a08a6c209d426f6158bd24b574f4f7b7aa0099c67c14a1f693b4dd04d0", data["mixdigest"])
}
if data["nonce"] != "0xf491f46b60fe04b3" {
t.Fatalf("Wrong nonce\r\nexpected %s\r\ngot %s", "0xf491f46b60fe04b3", data["nonce"])
}
if parseFloat(data["number"]) != "999999" {
t.Fatalf("Wrong block number\r\nexpected %s\r\ngot %s", "999999", parseFloat(data["number"]))
}
if parseMapElement(data["parent"]) != "bagiacgza2m6j3xu774hlvjxhd2fsnuv5ufom6ei4ply3mm3jrleeozt7b62a" {
t.Fatalf("Wrong Parent cid\r\nexpected %s\r\ngot %s", "bagiacgza2m6j3xu774hlvjxhd2fsnuv5ufom6ei4ply3mm3jrleeozt7b62a", parseMapElement(data["parent"]))
}
if parseMapElement(data["receipts"]) != "bagkacgzap6qpnsrkagbdecgybaa63ljx4pr2aa5vlsetdg2f5mpzpbrk2iuq" {
t.Fatalf("Wrong Receipt root cid\r\nexpected %s\r\ngot %s", "bagkacgzap6qpnsrkagbdecgybaa63ljx4pr2aa5vlsetdg2f5mpzpbrk2iuq", parseMapElement(data["receipts"]))
}
if parseMapElement(data["root"]) != "baglacgza5wmkus23dhec7m2tmtyikcfobjw6yzs7uv3ghxfjjroxavkm3yia" {
t.Fatalf("Wrong root hash cid\r\nexpected %s\r\ngot %s", "baglacgza5wmkus23dhec7m2tmtyikcfobjw6yzs7uv3ghxfjjroxavkm3yia", parseMapElement(data["root"]))
}
if parseFloat(data["time"]) != "1455404037" {
t.Fatalf("Wrong Time\r\nexpected %s\r\ngot %s", "1455404037", parseFloat(data["time"]))
}
if parseMapElement(data["tx"]) != "bagjacgzair6l3dci6smknejlccbrzx7vtr737s56onoksked2t5anxgxvzka" {
t.Fatalf("Wrong Tx root cid\r\nexpected %s\r\ngot %s", "bagjacgzair6l3dci6smknejlccbrzx7vtr737s56onoksked2t5anxgxvzka", parseMapElement(data["tx"]))
}
if parseMapElement(data["uncles"]) != "bagiqcgzadxge32g6y5oxvk4fwvt3ntgudljreri3ssfhie7qufbp2qgusndq" {
t.Fatalf("Wrong Uncle hash cid\r\nexpected %s\r\ngot %s", "bagiqcgzadxge32g6y5oxvk4fwvt3ntgudljreri3ssfhie7qufbp2qgusndq", parseMapElement(data["uncles"]))
}
}
func TestEthBlockLinks(t *testing.T) {
ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t)
links := ethBlock.Links()
if links[0].Cid.String() != "bagiacgza2m6j3xu774hlvjxhd2fsnuv5ufom6ei4ply3mm3jrleeozt7b62a" {
t.Fatalf("Wrong cid for parent link\r\nexpected: %s\r\ngot %s", "bagiacgza2m6j3xu774hlvjxhd2fsnuv5ufom6ei4ply3mm3jrleeozt7b62a", links[0].Cid.String())
}
if links[1].Cid.String() != "bagkacgzap6qpnsrkagbdecgybaa63ljx4pr2aa5vlsetdg2f5mpzpbrk2iuq" {
t.Fatalf("Wrong cid for receipt root link\r\nexpected: %s\r\ngot %s", "bagkacgzap6qpnsrkagbdecgybaa63ljx4pr2aa5vlsetdg2f5mpzpbrk2iuq", links[1].Cid.String())
}
if links[2].Cid.String() != "baglacgza5wmkus23dhec7m2tmtyikcfobjw6yzs7uv3ghxfjjroxavkm3yia" {
t.Fatalf("Wrong cid for state root link\r\nexpected: %s\r\ngot %s", "baglacgza5wmkus23dhec7m2tmtyikcfobjw6yzs7uv3ghxfjjroxavkm3yia", links[2].Cid.String())
}
if links[3].Cid.String() != "bagjacgzair6l3dci6smknejlccbrzx7vtr737s56onoksked2t5anxgxvzka" {
t.Fatalf("Wrong cid for tx root link\r\nexpected: %s\r\ngot %s", "bagjacgzair6l3dci6smknejlccbrzx7vtr737s56onoksked2t5anxgxvzka", links[3].Cid.String())
}
if links[4].Cid.String() != "bagiqcgzadxge32g6y5oxvk4fwvt3ntgudljreri3ssfhie7qufbp2qgusndq" {
t.Fatalf("Wrong cid for uncles root link\r\nexpected: %s\r\ngot %s", "bagiqcgzadxge32g6y5oxvk4fwvt3ntgudljreri3ssfhie7qufbp2qgusndq", links[4].Cid.String())
}
}
func TestEthBlockResolveEmptyPath(t *testing.T) {
ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t)
obj, rest, err := ethBlock.Resolve([]string{})
checkError(err, t)
if ethBlock != obj.(*EthHeader) {
t.Fatal("Should have returned the same eth-block object")
}
if len(rest) != 0 {
t.Fatalf("Wrong len of rest of the path returned\r\nexpected %d\r\ngot %d", 0, len(rest))
}
}
func TestEthBlockResolveNoSuchLink(t *testing.T) {
ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t)
_, _, err := ethBlock.Resolve([]string{"wewonthavethisfieldever"})
if err == nil {
t.Fatal("Should have failed with unknown field")
}
if err != ErrInvalidLink {
t.Fatalf("Wrong error message\r\nexpected %s\r\ngot %s", ErrInvalidLink, err.Error())
}
}
func TestEthBlockResolveBloom(t *testing.T) {
ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t)
obj, rest, err := ethBlock.Resolve([]string{"bloom"})
checkError(err, t)
// The marshaler of types.Bloom should output it as 0x
bloomInText := fmt.Sprintf("%x", obj.(types.Bloom))
if bloomInText[:10] != "0000000000" {
t.Fatalf("Wrong Bloom\r\nexpected %s\r\ngot %s", "0000000000", bloomInText[:10])
}
if len(rest) != 0 {
t.Fatalf("Wrong len of rest of the path returned\r\nexpected %d\r\ngot %d", 0, len(rest))
}
}
func TestEthBlockResolveBloomExtraPathElements(t *testing.T) {
ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t)
obj, rest, err := ethBlock.Resolve([]string{"bloom", "unexpected", "extra", "elements"})
if obj != nil {
t.Fatal("Returned obj should be nil")
}
if rest != nil {
t.Fatal("Returned rest should be nil")
}
if err.Error() != "unexpected path elements past bloom" {
t.Fatalf("Wrong error\r\nexpected %s\r\ngot %s", "unexpected path elements past bloom", err.Error())
}
}
func TestEthBlockResolveNonLinkFields(t *testing.T) {
ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t)
testCases := map[string][]string{
"coinbase": {"%x", "52bc44d5378309ee2abf1539bf71de1b7d7be3b5"},
"difficulty": {"%s", "12555463106190"},
"extra": {"%s", "0xd783010303844765746887676f312e342e32856c696e7578"},
"gaslimit": {"%d", "3141592"},
"gasused": {"%d", "231000"},
"mixdigest": {"%x", "5b10f4a08a6c209d426f6158bd24b574f4f7b7aa0099c67c14a1f693b4dd04d0"},
"nonce": {"%x", "f491f46b60fe04b3"},
"number": {"%s", "999999"},
"time": {"%d", "1455404037"},
}
for field, value := range testCases {
obj, rest, err := ethBlock.Resolve([]string{field})
checkError(err, t)
format := value[0]
result := value[1]
if fmt.Sprintf(format, obj) != result {
t.Fatalf("Wrong %v\r\nexpected %v\r\ngot %s", field, result, fmt.Sprintf(format, obj))
}
if len(rest) != 0 {
t.Fatalf("Wrong len of rest of the path returned\r\nexpected %d\r\ngot %d", 0, len(rest))
}
}
}
func TestEthBlockResolveNonLinkFieldsExtraPathElements(t *testing.T) {
ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t)
testCases := []string{
"coinbase",
"difficulty",
"extra",
"gaslimit",
"gasused",
"mixdigest",
"nonce",
"number",
"time",
}
for _, field := range testCases {
obj, rest, err := ethBlock.Resolve([]string{field, "unexpected", "extra", "elements"})
if obj != nil {
t.Fatal("Returned obj should be nil")
}
if rest != nil {
t.Fatal("Returned rest should be nil")
}
if err.Error() != "unexpected path elements past "+field {
t.Fatalf("Wrong error\r\nexpected %s\r\ngot %s", "unexpected path elements past "+field, err.Error())
}
}
}
func TestEthBlockResolveLinkFields(t *testing.T) {
ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t)
testCases := map[string]string{
"parent": "bagiacgza2m6j3xu774hlvjxhd2fsnuv5ufom6ei4ply3mm3jrleeozt7b62a",
"receipts": "bagkacgzap6qpnsrkagbdecgybaa63ljx4pr2aa5vlsetdg2f5mpzpbrk2iuq",
"root": "baglacgza5wmkus23dhec7m2tmtyikcfobjw6yzs7uv3ghxfjjroxavkm3yia",
"tx": "bagjacgzair6l3dci6smknejlccbrzx7vtr737s56onoksked2t5anxgxvzka",
"uncles": "bagiqcgzadxge32g6y5oxvk4fwvt3ntgudljreri3ssfhie7qufbp2qgusndq",
}
for field, result := range testCases {
obj, rest, err := ethBlock.Resolve([]string{field, "anything", "goes", "here"})
checkError(err, t)
lnk, ok := obj.(*node.Link)
if !ok {
t.Fatal("Returned object is not a link")
}
if lnk.Cid.String() != result {
t.Fatalf("Wrong %s cid\r\nexpected %v\r\ngot %v", field, result, lnk.Cid.String())
}
for i, p := range []string{"anything", "goes", "here"} {
if rest[i] != p {
t.Fatalf("Wrong rest of the path returned\r\nexpected %s\r\ngot %s", p, rest[i])
}
}
}
}
func TestEthBlockTreeBadParams(t *testing.T) {
ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t)
tree := ethBlock.Tree("non-empty-string", 0)
if tree != nil {
t.Fatal("Expected nil to be returned")
}
tree = ethBlock.Tree("non-empty-string", 1)
if tree != nil {
t.Fatal("Expected nil to be returned")
}
tree = ethBlock.Tree("", 0)
if tree != nil {
t.Fatal("Expected nil to be returned")
}
}
func TestEThBlockTree(t *testing.T) {
ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t)
tree := ethBlock.Tree("", 1)
lookupElements := map[string]interface{}{
"bloom": nil,
"coinbase": nil,
"difficulty": nil,
"extra": nil,
"gaslimit": nil,
"gasused": nil,
"mixdigest": nil,
"nonce": nil,
"number": nil,
"parent": nil,
"receipts": nil,
"root": nil,
"time": nil,
"tx": nil,
"uncles": nil,
}
if len(tree) != len(lookupElements) {
t.Fatalf("Wrong number of elements\r\nexpected %d\r\ngot %d", len(lookupElements), len(tree))
}
for _, te := range tree {
if _, ok := lookupElements[te]; !ok {
t.Fatalf("Unexpected Element: %v", te)
}
}
}
/*
The two functions above: TestEthBlockResolveNonLinkFields and
TestEthBlockResolveLinkFields did all the heavy lifting. Then, we will
just test two use cases.
*/
func TestEthBlockResolveLinksBadLink(t *testing.T) {
ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t)
obj, rest, err := ethBlock.ResolveLink([]string{"supercalifragilist"})
if obj != nil {
t.Fatalf("Expected obj to be nil")
}
if rest != nil {
t.Fatal("Expected rest to be nil")
}
if err != ErrInvalidLink {
t.Fatalf("Expected error\r\nexpected %s\r\ngot %s", ErrInvalidLink, err)
}
}
func TestEthBlockResolveLinksGoodLink(t *testing.T) {
ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t)
obj, rest, err := ethBlock.ResolveLink([]string{"tx", "0", "0", "0"})
if obj == nil {
t.Fatalf("Expected valid *node.Link obj to be returned")
}
if rest == nil {
t.Fatal("Expected rest to be returned")
}
for i, p := range []string{"0", "0", "0"} {
if rest[i] != p {
t.Fatalf("Wrong rest of the path returned\r\nexpected %s\r\ngot %s", p, rest[i])
}
}
if err != nil {
t.Fatal("Non error expected")
}
}
/*
These functions below should go away
We are working on test coverage anyways...
*/
func TestEthBlockCopy(t *testing.T) {
ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t)
defer func() {
r := recover()
if r == nil {
t.Fatal("Expected panic")
}
if r != "implement me" {
t.Fatalf("Wrong panic message\r\nexpected %s\r\ngot %s", "'implement me'", r)
}
}()
_ = ethBlock.Copy()
}
func TestEthBlockStat(t *testing.T) {
ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t)
obj, err := ethBlock.Stat()
if obj == nil {
t.Fatal("Expected a not null object node.NodeStat")
}
if err != nil {
t.Fatal("Expected a nil error")
}
}
func TestEthBlockSize(t *testing.T) {
ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t)
size, err := ethBlock.Size()
if size != 0 {
t.Fatalf("Wrong size\r\nexpected %d\r\ngot %d", 0, size)
}
if err != nil {
t.Fatal("Expected a nil error")
}
}
/*
AUXILIARS
*/
// checkError makes 3 lines into 1.
func checkError(err error, t *testing.T) {
if err != nil {
_, fn, line, _ := runtime.Caller(1)
t.Fatalf("[%v:%v] %v", fn, line, err)
}
}
// parseFloat is a convenience function to test json output
func parseFloat(v interface{}) string {
return strconv.FormatFloat(v.(float64), 'f', 0, 64)
}
// parseMapElement is a convenience function to tets json output
func parseMapElement(v interface{}) string {
return v.(map[string]interface{})["/"].(string)
}
// prepareStoredEthBlock reads the block from a file source to get its rawdata
// and computes its cid, for then, feeding it into a new IPLD block function.
// So we can pretend that we got this block from the datastore
func prepareStoredEthBlock(filepath string, t *testing.T) *block.BasicBlock {
// Prepare the "fetched block". This one is supposed to be in the datastore
// and given away by github.com/ipfs/go-ipfs/merkledag
fi, err := os.Open(filepath)
checkError(err, t)
b, err := ioutil.ReadAll(fi)
checkError(err, t)
c, err := RawdataToCid(MEthHeader, b, multihash.KECCAK_256)
checkError(err, t)
// It's good to clarify that this one below is an IPLD block
storedEthBlock, err := block.NewBlockWithCid(b, c)
checkError(err, t)
return storedEthBlock
}
// prepareDecodedEthBlock is more complex than function above, as it stores a
// basic block and RLP-decodes it
func prepareDecodedEthBlock(filepath string, t *testing.T) *EthHeader {
// Get the block from the datastore and decode it.
storedEthBlock := prepareStoredEthBlock("test_data/eth-block-header-rlp-999999", t)
ethBlock, err := DecodeEthHeader(storedEthBlock.Cid(), storedEthBlock.RawData())
checkError(err, t)
return ethBlock
}
// testEthBlockFields checks the fields of EthBlock one by one.
func testEthBlockFields(ethBlock *EthHeader, t *testing.T) {
// Was the cid calculated?
if ethBlock.Cid().String() != "bagiacgzawt5236hkiuvrhfyy4jya3qitlt6icfcqgheew6vsptlraokppm4a" {
t.Fatalf("Wrong cid\r\nexpected %s\r\ngot %s", "bagiacgzawt5236hkiuvrhfyy4jya3qitlt6icfcqgheew6vsptlraokppm4a", ethBlock.Cid().String())
}
// Do we have the rawdata available?
if fmt.Sprintf("%x", ethBlock.RawData()[:10]) != "f90218a0d33c9dde9fff" {
t.Fatalf("Wrong Rawdata\r\nexpected %s\r\ngot %s", "f90218a0d33c9dde9fff", fmt.Sprintf("%x", ethBlock.RawData()[:10]))
}
// Proper Fields of types.Header
if fmt.Sprintf("%x", ethBlock.ParentHash) != "d33c9dde9fff0ebaa6e71e8b26d2bda15ccf111c7af1b633698ac847667f0fb4" {
t.Fatalf("Wrong ParentHash\r\nexpected %s\r\ngot %s", "d33c9dde9fff0ebaa6e71e8b26d2bda15ccf111c7af1b633698ac847667f0fb4", fmt.Sprintf("%x", ethBlock.ParentHash))
}
if fmt.Sprintf("%x", ethBlock.UncleHash) != "1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347" {
t.Fatalf("Wrong UncleHash field\r\nexpected %s\r\ngot %s", "1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", fmt.Sprintf("%x", ethBlock.UncleHash))
}
if fmt.Sprintf("%x", ethBlock.Coinbase) != "52bc44d5378309ee2abf1539bf71de1b7d7be3b5" {
t.Fatalf("Wrong Coinbase\r\nexpected %s\r\ngot %s", "52bc44d5378309ee2abf1539bf71de1b7d7be3b5", fmt.Sprintf("%x", ethBlock.Coinbase))
}
if fmt.Sprintf("%x", ethBlock.Root) != "ed98aa4b5b19c82fb35364f08508ae0a6dec665fa57663dca94c5d70554cde10" {
t.Fatalf("Wrong Root\r\nexpected %s\r\ngot %s", "ed98aa4b5b19c82fb35364f08508ae0a6dec665fa57663dca94c5d70554cde10", fmt.Sprintf("%x", ethBlock.Root))
}
if fmt.Sprintf("%x", ethBlock.TxHash) != "447cbd8c48f498a6912b10831cdff59c7fbfcbbe735ca92883d4fa06dcd7ae54" {
t.Fatalf("Wrong TxHash\r\nexpected %s\r\ngot %s", "447cbd8c48f498a6912b10831cdff59c7fbfcbbe735ca92883d4fa06dcd7ae54", fmt.Sprintf("%x", ethBlock.TxHash))
}
if fmt.Sprintf("%x", ethBlock.ReceiptHash) != "7fa0f6ca2a01823208d80801edad37e3e3a003b55c89319b45eb1f97862ad229" {
t.Fatalf("Wrong ReceiptHash\r\nexpected %s\r\ngot %s", "7fa0f6ca2a01823208d80801edad37e3e3a003b55c89319b45eb1f97862ad229", fmt.Sprintf("%x", ethBlock.ReceiptHash))
}
if len(ethBlock.Bloom) != 256 {
t.Fatalf("Wrong Bloom Length\r\nexpected %d\r\ngot %d", 256, len(ethBlock.Bloom))
}
if fmt.Sprintf("%x", ethBlock.Bloom[71:76]) != "0000000000" { // You wouldn't want me to print out the whole bloom field?
t.Fatalf("Wrong Bloom\r\nexpected %s\r\ngot %s", "0000000000", fmt.Sprintf("%x", ethBlock.Bloom[71:76]))
}
if ethBlock.Difficulty.String() != "12555463106190" {
t.Fatalf("Wrong Difficulty\r\nexpected %s\r\ngot %s", "12555463106190", ethBlock.Difficulty.String())
}
if ethBlock.Number.String() != "999999" {
t.Fatalf("Wrong Block Number\r\nexpected %s\r\ngot %s", "999999", ethBlock.Number.String())
}
if ethBlock.GasLimit != uint64(3141592) {
t.Fatalf("Wrong Gas Limit\r\nexpected %d\r\ngot %d", 3141592, ethBlock.GasLimit)
}
if ethBlock.GasUsed != uint64(231000) {
t.Fatalf("Wrong Gas Used\r\nexpected %d\r\ngot %d", 231000, ethBlock.GasUsed)
}
if ethBlock.Time != uint64(1455404037) {
t.Fatalf("Wrong Time\r\nexpected %d\r\ngot %d", 1455404037, ethBlock.Time)
}
if fmt.Sprintf("%x", ethBlock.Extra) != "d783010303844765746887676f312e342e32856c696e7578" {
t.Fatalf("Wrong Extra\r\nexpected %s\r\ngot %s", "d783010303844765746887676f312e342e32856c696e7578", fmt.Sprintf("%x", ethBlock.Extra))
}
if fmt.Sprintf("%x", ethBlock.Nonce) != "f491f46b60fe04b3" {
t.Fatalf("Wrong Nonce\r\nexpected %s\r\ngot %s", "f491f46b60fe04b3", fmt.Sprintf("%x", ethBlock.Nonce))
}
if fmt.Sprintf("%x", ethBlock.MixDigest) != "5b10f4a08a6c209d426f6158bd24b574f4f7b7aa0099c67c14a1f693b4dd04d0" {
t.Fatalf("Wrong MixDigest\r\nexpected %s\r\ngot %s", "5b10f4a08a6c209d426f6158bd24b574f4f7b7aa0099c67c14a1f693b4dd04d0", fmt.Sprintf("%x", ethBlock.MixDigest))
}
}

157
statediff/indexer/ipfs/ipld/eth_log.go Normal file
View File

@ -0,0 +1,157 @@
package ipld
import (
"fmt"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ipfs/go-cid"
node "github.com/ipfs/go-ipld-format"
mh "github.com/multiformats/go-multihash"
)
// EthLog (eth-log, codec 0x9a), represents an ethereum block header
type EthLog struct {
*types.Log
rawData []byte
cid cid.Cid
}
// Static (compile time) check that EthLog satisfies the node.Node interface.
var _ node.Node = (*EthLog)(nil)
// NewLog create a new EthLog IPLD node
func NewLog(log *types.Log) (*EthLog, error) {
logRaw, err := rlp.EncodeToBytes(log)
if err != nil {
return nil, err
}
c, err := RawdataToCid(MEthLog, logRaw, mh.KECCAK_256)
if err != nil {
return nil, err
}
return &EthLog{
Log: log,
cid: c,
rawData: logRaw,
}, nil
}
// DecodeEthLogs takes a cid and its raw binary data
func DecodeEthLogs(c cid.Cid, b []byte) (*EthLog, error) {
l := new(types.Log)
if err := rlp.DecodeBytes(b, l); err != nil {
return nil, err
}
return &EthLog{
Log: l,
cid: c,
rawData: b,
}, nil
}
/*
Block INTERFACE
*/
// RawData returns the binary of the RLP encode of the log.
func (l *EthLog) RawData() []byte {
return l.rawData
}
// Cid returns the cid of the receipt log.
func (l *EthLog) Cid() cid.Cid {
return l.cid
}
// String is a helper for output
func (l *EthLog) String() string {
return fmt.Sprintf("<EthereumLog %s>", l.cid)
}
// Loggable returns in a map the type of IPLD Link.
func (l *EthLog) Loggable() map[string]interface{} {
return map[string]interface{}{
"type": "eth-log",
}
}
// Resolve resolves a path through this node, stopping at any link boundary
// and returning the object found as well as the remaining path to traverse
func (l *EthLog) Resolve(p []string) (interface{}, []string, error) {
if len(p) == 0 {
return l, nil, nil
}
if len(p) > 1 {
return nil, nil, fmt.Errorf("unexpected path elements past %s", p[0])
}
switch p[0] {
case "address":
return l.Address, nil, nil
case "data":
// This is a []byte. By default they are marshalled into Base64.
return fmt.Sprintf("0x%x", l.Data), nil, nil
case "topics":
return l.Topics, nil, nil
case "logIndex":
return l.Index, nil, nil
case "removed":
return l.Removed, nil, nil
default:
return nil, nil, ErrInvalidLink
}
}
// Tree lists all paths within the object under 'path', and up to the given depth.
// To list the entire object (similar to `find .`) pass "" and -1
func (l *EthLog) Tree(p string, depth int) []string {
if p != "" || depth == 0 {
return nil
}
return []string{
"address",
"data",
"topics",
"logIndex",
"removed",
}
}
// ResolveLink is a helper function that calls resolve and asserts the
// output is a link
func (l *EthLog) ResolveLink(p []string) (*node.Link, []string, error) {
obj, rest, err := l.Resolve(p)
if err != nil {
return nil, nil, err
}
if lnk, ok := obj.(*node.Link); ok {
return lnk, rest, nil
}
return nil, nil, fmt.Errorf("resolved item was not a link")
}
// Copy will go away. It is here to comply with the Node interface.
func (l *EthLog) Copy() node.Node {
panic("implement me")
}
// Links is a helper function that returns all links within this object
func (l *EthLog) Links() []*node.Link {
return nil
}
// Stat will go away. It is here to comply with the interface.
func (l *EthLog) Stat() (*node.NodeStat, error) {
return &node.NodeStat{}, nil
}
// Size will go away. It is here to comply with the interface.
func (l *EthLog) Size() (uint64, error) {
return 0, nil
}

144
statediff/indexer/ipfs/ipld/eth_log_trie.go Normal file
View File

@ -0,0 +1,144 @@
package ipld
import (
"fmt"
node "github.com/ipfs/go-ipld-format"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ipfs/go-cid"
"github.com/multiformats/go-multihash"
)
// EthLogTrie (eth-tx-trie codec 0x9p) represents
// a node from the transaction trie in ethereum.
type EthLogTrie struct {
*TrieNode
}
/*
OUTPUT
*/
// DecodeEthLogTrie returns an EthLogTrie object from its cid and rawdata.
func DecodeEthLogTrie(c cid.Cid, b []byte) (*EthLogTrie, error) {
tn, err := decodeTrieNode(c, b, decodeEthLogTrieLeaf)
if err != nil {
return nil, err
}
return &EthLogTrie{TrieNode: tn}, nil
}
// decodeEthLogTrieLeaf parses a eth-log-trie leaf
// from decoded RLP elements
func decodeEthLogTrieLeaf(i []interface{}) ([]interface{}, error) {
l := new(types.Log)
if err := rlp.DecodeBytes(i[1].([]byte), l); err != nil {
return nil, err
}
c, err := RawdataToCid(MEthLogTrie, i[1].([]byte), multihash.KECCAK_256)
if err != nil {
return nil, err
}
return []interface{}{
i[0].([]byte),
&EthLog{
Log: l,
cid: c,
rawData: i[1].([]byte),
},
}, nil
}
/*
Block INTERFACE
*/
// RawData returns the binary of the RLP encode of the transaction.
func (t *EthLogTrie) RawData() []byte {
return t.rawdata
}
// Cid returns the cid of the transaction.
func (t *EthLogTrie) Cid() cid.Cid {
return t.cid
}
// String is a helper for output
func (t *EthLogTrie) String() string {
return fmt.Sprintf("<EthereumLogTrie %s>", t.cid)
}
// Loggable returns in a map the type of IPLD Link.
func (t *EthLogTrie) Loggable() map[string]interface{} {
return map[string]interface{}{
"type": "eth-log-trie",
}
}
// logTrie wraps a localTrie for use on the receipt trie.
type logTrie struct {
*localTrie
}
// newLogTrie initializes and returns a logTrie.
func newLogTrie() *logTrie {
return &logTrie{
localTrie: newLocalTrie(),
}
}
// getNodes invokes the localTrie, which computes the root hash of the
// log trie and returns its database keys, to return a slice
// of EthLogTrie nodes.
func (rt *logTrie) getNodes() ([]node.Node, error) {
keys, err := rt.getKeys()
if err != nil {
return nil, err
}
out := make([]node.Node, 0, len(keys))
for _, k := range keys {
n, err := rt.getNodeFromDB(k)
if err != nil {
return nil, err
}
out = append(out, n)
}
return out, nil
}
func (rt *logTrie) getNodeFromDB(key []byte) (*EthLogTrie, error) {
rawdata, err := rt.db.Get(key)
if err != nil {
return nil, err
}
tn := &TrieNode{
cid: keccak256ToCid(MEthLogTrie, key),
rawdata: rawdata,
}
return &EthLogTrie{TrieNode: tn}, nil
}
// getLeafNodes invokes the localTrie, which returns a slice
// of EthLogTrie leaf nodes.
func (rt *logTrie) getLeafNodes() ([]*EthLogTrie, []*nodeKey, error) {
keys, err := rt.getLeafKeys()
if err != nil {
return nil, nil, err
}
out := make([]*EthLogTrie, 0, len(keys))
for _, k := range keys {
n, err := rt.getNodeFromDB(k.dbKey)
if err != nil {
return nil, nil, err
}
out = append(out, n)
}
return out, keys, nil
}

302
statediff/indexer/ipfs/ipld/eth_parser.go Normal file
View File

@ -0,0 +1,302 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipld
import (
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
node "github.com/ipfs/go-ipld-format"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ipfs/go-cid"
"github.com/multiformats/go-multihash"
)
// FromBlockRLP takes an RLP message representing
// an ethereum block header or body (header, ommers and txs)
// to return it as a set of IPLD nodes for further processing.
func FromBlockRLP(r io.Reader) (*EthHeader, []*EthTx, []*EthTxTrie, error) {
// We may want to use this stream several times
rawdata, err := ioutil.ReadAll(r)
if err != nil {
return nil, nil, nil, err
}
// Let's try to decode the received element as a block body
var decodedBlock types.Block
err = rlp.Decode(bytes.NewBuffer(rawdata), &decodedBlock)
if err != nil {
if err.Error()[:41] != "rlp: expected input list for types.Header" {
return nil, nil, nil, err
}
// Maybe it is just a header... (body sans ommers and txs)
var decodedHeader types.Header
err := rlp.Decode(bytes.NewBuffer(rawdata), &decodedHeader)
if err != nil {
return nil, nil, nil, err
}
c, err := RawdataToCid(MEthHeader, rawdata, multihash.KECCAK_256)
if err != nil {
return nil, nil, nil, err
}
// It was a header
return &EthHeader{
Header: &decodedHeader,
cid: c,
rawdata: rawdata,
}, nil, nil, nil
}
// This is a block body (header + ommers + txs)
// We'll extract the header bits here
headerRawData := getRLP(decodedBlock.Header())
c, err := RawdataToCid(MEthHeader, headerRawData, multihash.KECCAK_256)
if err != nil {
return nil, nil, nil, err
}
ethBlock := &EthHeader{
Header: decodedBlock.Header(),
cid: c,
rawdata: headerRawData,
}
// Process the found eth-tx objects
ethTxNodes, ethTxTrieNodes, err := processTransactions(decodedBlock.Transactions(),
decodedBlock.Header().TxHash[:])
if err != nil {
return nil, nil, nil, err
}
return ethBlock, ethTxNodes, ethTxTrieNodes, nil
}
// FromBlockJSON takes the output of an ethereum client JSON API
// (i.e. parity or geth) and returns a set of IPLD nodes.
func FromBlockJSON(r io.Reader) (*EthHeader, []*EthTx, []*EthTxTrie, error) {
var obj objJSONHeader
dec := json.NewDecoder(r)
err := dec.Decode(&obj)
if err != nil {
return nil, nil, nil, err
}
headerRawData := getRLP(obj.Result.Header)
c, err := RawdataToCid(MEthHeader, headerRawData, multihash.KECCAK_256)
if err != nil {
return nil, nil, nil, err
}
ethBlock := &EthHeader{
Header: &obj.Result.Header,
cid: c,
rawdata: headerRawData,
}
// Process the found eth-tx objects
ethTxNodes, ethTxTrieNodes, err := processTransactions(obj.Result.Transactions,
obj.Result.Header.TxHash[:])
if err != nil {
return nil, nil, nil, err
}
return ethBlock, ethTxNodes, ethTxTrieNodes, nil
}
// FromBlockAndReceipts takes a block and processes it
// to return it a set of IPLD nodes for further processing.
func FromBlockAndReceipts(block *types.Block, receipts []*types.Receipt) (*EthHeader, []*EthHeader, []*EthTx, []*EthTxTrie, []*EthReceipt, []*EthRctTrie, [][]node.Node, [][]cid.Cid, []cid.Cid, error) {
// Process the header
headerNode, err := NewEthHeader(block.Header())
if err != nil {
return nil, nil, nil, nil, nil, nil, nil, nil, nil, err
}
// Process the uncles
uncleNodes := make([]*EthHeader, len(block.Uncles()))
for i, uncle := range block.Uncles() {
uncleNode, err := NewEthHeader(uncle)
if err != nil {
return nil, nil, nil, nil, nil, nil, nil, nil, nil, err
}
uncleNodes[i] = uncleNode
}
// Process the txs
txNodes, txTrieNodes, err := processTransactions(block.Transactions(),
block.Header().TxHash[:])
if err != nil {
return nil, nil, nil, nil, nil, nil, nil, nil, nil, err
}
// Process the receipts and logs
rctNodes, tctTrieNodes, logTrieAndLogNodes, logLeafNodeCIDs, rctLeafNodeCIDs, err := processReceiptsAndLogs(receipts,
block.Header().ReceiptHash[:])
return headerNode, uncleNodes, txNodes, txTrieNodes, rctNodes, tctTrieNodes, logTrieAndLogNodes, logLeafNodeCIDs, rctLeafNodeCIDs, err
}
// processTransactions will take the found transactions in a parsed block body
// to return IPLD node slices for eth-tx and eth-tx-trie
func processTransactions(txs []*types.Transaction, expectedTxRoot []byte) ([]*EthTx, []*EthTxTrie, error) {
var ethTxNodes []*EthTx
transactionTrie := newTxTrie()
for idx, tx := range txs {
ethTx, err := NewEthTx(tx)
if err != nil {
return nil, nil, err
}
ethTxNodes = append(ethTxNodes, ethTx)
if err := transactionTrie.Add(idx, ethTx.RawData()); err != nil {
return nil, nil, err
}
}
if !bytes.Equal(transactionTrie.rootHash(), expectedTxRoot) {
return nil, nil, fmt.Errorf("wrong transaction hash computed")
}
txTrieNodes, err := transactionTrie.getNodes()
return ethTxNodes, txTrieNodes, err
}
// processReceiptsAndLogs will take in receipts
// to return IPLD node slices for eth-rct, eth-rct-trie, eth-log, eth-log-trie, eth-log-trie-CID, eth-rct-trie-CID
func processReceiptsAndLogs(rcts []*types.Receipt, expectedRctRoot []byte) ([]*EthReceipt, []*EthRctTrie, [][]node.Node, [][]cid.Cid, []cid.Cid, error) {
// Pre allocating memory.
ethRctNodes := make([]*EthReceipt, 0, len(rcts))
ethLogleafNodeCids := make([][]cid.Cid, 0, len(rcts))
ethLogTrieAndLogNodes := make([][]node.Node, 0, len(rcts))
receiptTrie := NewRctTrie()
for idx, rct := range rcts {
// Process logs for each receipt.
logTrieNodes, leafNodeCids, logTrieHash, err := processLogs(rct.Logs)
if err != nil {
return nil, nil, nil, nil, nil, err
}
rct.LogRoot = logTrieHash
ethLogTrieAndLogNodes = append(ethLogTrieAndLogNodes, logTrieNodes)
ethLogleafNodeCids = append(ethLogleafNodeCids, leafNodeCids)
ethRct, err := NewReceipt(rct)
if err != nil {
return nil, nil, nil, nil, nil, err
}
ethRctNodes = append(ethRctNodes, ethRct)
if err = receiptTrie.Add(idx, ethRct.RawData()); err != nil {
return nil, nil, nil, nil, nil, err
}
}
if !bytes.Equal(receiptTrie.rootHash(), expectedRctRoot) {
return nil, nil, nil, nil, nil, fmt.Errorf("wrong receipt hash computed")
}
rctTrieNodes, err := receiptTrie.GetNodes()
if err != nil {
return nil, nil, nil, nil, nil, err
}
rctLeafNodes, keys, err := receiptTrie.GetLeafNodes()
if err != nil {
return nil, nil, nil, nil, nil, err
}
ethRctleafNodeCids := make([]cid.Cid, len(rctLeafNodes))
for i, rln := range rctLeafNodes {
var idx uint
r := bytes.NewReader(keys[i].TrieKey)
err = rlp.Decode(r, &idx)
if err != nil {
return nil, nil, nil, nil, nil, err
}
ethRctleafNodeCids[idx] = rln.Cid()
}
return ethRctNodes, rctTrieNodes, ethLogTrieAndLogNodes, ethLogleafNodeCids, ethRctleafNodeCids, err
}
const keccak256Length = 32
func processLogs(logs []*types.Log) ([]node.Node, []cid.Cid, common.Hash, error) {
logTr := newLogTrie()
shortLog := make(map[uint64]*EthLog, len(logs))
for idx, log := range logs {
logRaw, err := rlp.EncodeToBytes(log)
if err != nil {
return nil, nil, common.Hash{}, err
}
// if len(logRaw) <= keccak256Length it is possible this value's "leaf node"
// will be stored in its parent branch but only if len(partialPathOfTheNode) + len(logRaw) <= keccak256Length
// But we can't tell what the partial path will be until the trie is Commit()-ed
// So wait until we collect all the leaf nodes, and if we are missing any at the indexes we note in shortLogCIDs
// we know that these "leaf nodes" were internalized into their parent branch node and we move forward with
// using the cid.Cid we cached in shortLogCIDs
if len(logRaw) <= keccak256Length {
logNode, err := NewLog(log)
if err != nil {
return nil, nil, common.Hash{}, err
}
shortLog[uint64(idx)] = logNode
}
if err = logTr.Add(idx, logRaw); err != nil {
return nil, nil, common.Hash{}, err
}
}
logTrieAndLogNodes, err := logTr.getNodes()
if err != nil {
return nil, nil, common.Hash{}, err
}
leafNodes, keys, err := logTr.getLeafNodes()
if err != nil {
return nil, nil, common.Hash{}, err
}
leafNodeCids := make([]cid.Cid, len(logs))
for i, ln := range leafNodes {
var idx uint
r := bytes.NewReader(keys[i].TrieKey)
err = rlp.Decode(r, &idx)
if err != nil {
return nil, nil, common.Hash{}, err
}
leafNodeCids[idx] = ln.Cid()
}
// this is where we check which logs <= keccak256Length were actually internalized into parent branch node
// and replace those that were with the cid.Cid for the raw log IPLD
for i, l := range shortLog {
if !leafNodeCids[i].Defined() {
leafNodeCids[i] = l.Cid()
// if the leaf node was internalized, we append an IPLD for log itself to the list of IPLDs we need to publish
logTrieAndLogNodes = append(logTrieAndLogNodes, l)
}
}
return logTrieAndLogNodes, leafNodeCids, common.BytesToHash(logTr.rootHash()), err
}

107
statediff/indexer/ipfs/ipld/eth_parser_test.go Normal file
View File

@ -0,0 +1,107 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipld
import (
"io/ioutil"
"path/filepath"
"testing"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/statediff/indexer/mocks"
"github.com/stretchr/testify/require"
)
type kind string
const (
legacy kind = "legacy"
eip1559 kind = "eip2930"
)
var blockFileNames = []string{
"eth-block-12252078",
"eth-block-12365585",
"eth-block-12365586",
}
var receiptsFileNames = []string{
"eth-receipts-12252078",
"eth-receipts-12365585",
"eth-receipts-12365586",
}
var kinds = []kind{
eip1559,
eip1559,
legacy,
}
type testCase struct {
kind kind
block *types.Block
receipts types.Receipts
}
func loadBlockData(t *testing.T) []testCase {
fileDir := "./eip2930_test_data"
testCases := make([]testCase, len(blockFileNames))
for i, blockFileName := range blockFileNames {
blockRLP, err := ioutil.ReadFile(filepath.Join(fileDir, blockFileName))
if err != nil {
t.Fatalf("failed to load blockRLP from file, err %v", err)
}
block := new(types.Block)
if err := rlp.DecodeBytes(blockRLP, block); err != nil {
t.Fatalf("failed to decode blockRLP, err %v", err)
}
receiptsFileName := receiptsFileNames[i]
receiptsRLP, err := ioutil.ReadFile(filepath.Join(fileDir, receiptsFileName))
if err != nil {
t.Fatalf("failed to load receiptsRLP from file, err %s", err)
}
receipts := make(types.Receipts, 0)
if err := rlp.DecodeBytes(receiptsRLP, &receipts); err != nil {
t.Fatalf("failed to decode receiptsRLP, err %s", err)
}
testCases[i] = testCase{
block: block,
receipts: receipts,
kind: kinds[i],
}
}
return testCases
}
func TestFromBlockAndReceipts(t *testing.T) {
testCases := loadBlockData(t)
for _, tc := range testCases {
_, _, _, _, _, _, _, _, _, err := FromBlockAndReceipts(tc.block, tc.receipts)
if err != nil {
t.Fatalf("error generating IPLDs from block and receipts, err %v, kind %s, block hash %s", err, tc.kind, tc.block.Hash())
}
}
}
func TestProcessLogs(t *testing.T) {
logs := []*types.Log{mocks.MockLog1, mocks.MockLog2}
nodes, cids, _, err := processLogs(logs)
require.NoError(t, err)
require.GreaterOrEqual(t, len(nodes), len(logs))
require.Equal(t, len(logs), len(cids))
}

204
statediff/indexer/ipfs/ipld/eth_receipt.go Normal file
View File

@ -0,0 +1,204 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipld
import (
"encoding/json"
"fmt"
"strconv"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ipfs/go-cid"
node "github.com/ipfs/go-ipld-format"
mh "github.com/multiformats/go-multihash"
)
type EthReceipt struct {
*types.Receipt
rawdata []byte
cid cid.Cid
}
// Static (compile time) check that EthReceipt satisfies the node.Node interface.
var _ node.Node = (*EthReceipt)(nil)
/*
INPUT
*/
// NewReceipt converts a types.ReceiptForStorage to an EthReceipt IPLD node
func NewReceipt(receipt *types.Receipt) (*EthReceipt, error) {
rctRaw, err := receipt.MarshalBinary()
if err != nil {
return nil, err
}
c, err := RawdataToCid(MEthTxReceipt, rctRaw, mh.KECCAK_256)
if err != nil {
return nil, err
}
return &EthReceipt{
Receipt: receipt,
cid: c,
rawdata: rctRaw,
}, nil
}
/*
OUTPUT
*/
// DecodeEthReceipt takes a cid and its raw binary data
// from IPFS and returns an EthTx object for further processing.
func DecodeEthReceipt(c cid.Cid, b []byte) (*EthReceipt, error) {
r := new(types.Receipt)
if err := r.UnmarshalBinary(b); err != nil {
return nil, err
}
return &EthReceipt{
Receipt: r,
cid: c,
rawdata: b,
}, nil
}
/*
Block INTERFACE
*/
// RawData returns the binary of the RLP encode of the receipt.
func (r *EthReceipt) RawData() []byte {
return r.rawdata
}
// Cid returns the cid of the receipt.
func (r *EthReceipt) Cid() cid.Cid {
return r.cid
}
// String is a helper for output
func (r *EthReceipt) String() string {
return fmt.Sprintf("<EthereumReceipt %s>", r.cid)
}
// Loggable returns in a map the type of IPLD Link.
func (r *EthReceipt) Loggable() map[string]interface{} {
return map[string]interface{}{
"type": "eth-receipt",
}
}
// Resolve resolves a path through this node, stopping at any link boundary
// and returning the object found as well as the remaining path to traverse
func (r *EthReceipt) Resolve(p []string) (interface{}, []string, error) {
if len(p) == 0 {
return r, nil, nil
}
first, rest := p[0], p[1:]
if first != "logs" && len(p) != 1 {
return nil, nil, fmt.Errorf("unexpected path elements past %s", first)
}
switch first {
case "logs":
return &node.Link{Cid: commonHashToCid(MEthLog, r.LogRoot)}, rest, nil
case "root":
return r.PostState, nil, nil
case "status":
return r.Status, nil, nil
case "cumulativeGasUsed":
return r.CumulativeGasUsed, nil, nil
case "logsBloom":
return r.Bloom, nil, nil
case "transactionHash":
return r.TxHash, nil, nil
case "contractAddress":
return r.ContractAddress, nil, nil
case "gasUsed":
return r.GasUsed, nil, nil
case "type":
return r.Type, nil, nil
default:
return nil, nil, ErrInvalidLink
}
}
// Tree lists all paths within the object under 'path', and up to the given depth.
// To list the entire object (similar to `find .`) pass "" and -1
func (r *EthReceipt) Tree(p string, depth int) []string {
if p != "" || depth == 0 {
return nil
}
return []string{"type", "root", "status", "cumulativeGasUsed", "logsBloom", "logs", "transactionHash", "contractAddress", "gasUsed"}
}
// ResolveLink is a helper function that calls resolve and asserts the
// output is a link
func (r *EthReceipt) ResolveLink(p []string) (*node.Link, []string, error) {
obj, rest, err := r.Resolve(p)
if err != nil {
return nil, nil, err
}
if lnk, ok := obj.(*node.Link); ok {
return lnk, rest, nil
}
return nil, nil, fmt.Errorf("resolved item was not a link")
}
// Copy will go away. It is here to comply with the Node interface.
func (r *EthReceipt) Copy() node.Node {
panic("implement me")
}
// Links is a helper function that returns all links within this object
func (r *EthReceipt) Links() []*node.Link {
return []*node.Link{
{Cid: commonHashToCid(MEthLog, r.LogRoot)},
}
}
// Stat will go away. It is here to comply with the interface.
func (r *EthReceipt) Stat() (*node.NodeStat, error) {
return &node.NodeStat{}, nil
}
// Size will go away. It is here to comply with the interface.
func (r *EthReceipt) Size() (uint64, error) {
return strconv.ParseUint(r.Receipt.Size().String(), 10, 64)
}
/*
EthReceipt functions
*/
// MarshalJSON processes the receipt into readable JSON format.
func (r *EthReceipt) MarshalJSON() ([]byte, error) {
out := map[string]interface{}{
"root": r.PostState,
"status": r.Status,
"cumulativeGasUsed": r.CumulativeGasUsed,
"logsBloom": r.Bloom,
"logs": r.Logs,
"transactionHash": r.TxHash,
"contractAddress": r.ContractAddress,
"gasUsed": r.GasUsed,
}
return json.Marshal(out)
}

175
statediff/indexer/ipfs/ipld/eth_receipt_trie.go Normal file
View File

@ -0,0 +1,175 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipld
import (
"fmt"
"github.com/ipfs/go-cid"
node "github.com/ipfs/go-ipld-format"
"github.com/multiformats/go-multihash"
"github.com/ethereum/go-ethereum/core/types"
)
// EthRctTrie (eth-tx-trie codec 0x92) represents
// a node from the transaction trie in ethereum.
type EthRctTrie struct {
*TrieNode
}
// Static (compile time) check that EthRctTrie satisfies the node.Node interface.
var _ node.Node = (*EthRctTrie)(nil)
/*
INPUT
*/
// To create a proper trie of the eth-tx-trie objects, it is required
// to input all transactions belonging to a forest in a single step.
// We are adding the transactions, and creating its trie on
// block body parsing time.
/*
OUTPUT
*/
// DecodeEthRctTrie returns an EthRctTrie object from its cid and rawdata.
func DecodeEthRctTrie(c cid.Cid, b []byte) (*EthRctTrie, error) {
tn, err := decodeTrieNode(c, b, decodeEthRctTrieLeaf)
if err != nil {
return nil, err
}
return &EthRctTrie{TrieNode: tn}, nil
}
// decodeEthRctTrieLeaf parses a eth-rct-trie leaf
//from decoded RLP elements
func decodeEthRctTrieLeaf(i []interface{}) ([]interface{}, error) {
r := new(types.Receipt)
if err := r.UnmarshalBinary(i[1].([]byte)); err != nil {
return nil, err
}
c, err := RawdataToCid(MEthTxReceipt, i[1].([]byte), multihash.KECCAK_256)
if err != nil {
return nil, err
}
return []interface{}{
i[0].([]byte),
&EthReceipt{
Receipt: r,
cid: c,
rawdata: i[1].([]byte),
},
}, nil
}
/*
Block INTERFACE
*/
// RawData returns the binary of the RLP encode of the transaction.
func (t *EthRctTrie) RawData() []byte {
return t.rawdata
}
// Cid returns the cid of the transaction.
func (t *EthRctTrie) Cid() cid.Cid {
return t.cid
}
// String is a helper for output
func (t *EthRctTrie) String() string {
return fmt.Sprintf("<EthereumRctTrie %s>", t.cid)
}
// Loggable returns in a map the type of IPLD Link.
func (t *EthRctTrie) Loggable() map[string]interface{} {
return map[string]interface{}{
"type": "eth-rct-trie",
}
}
/*
EthRctTrie functions
*/
// rctTrie wraps a localTrie for use on the receipt trie.
type rctTrie struct {
*localTrie
}
// NewRctTrie initializes and returns a rctTrie.
func NewRctTrie() *rctTrie {
return &rctTrie{
localTrie: newLocalTrie(),
}
}
// GetNodes invokes the localTrie, which computes the root hash of the
// transaction trie and returns its database keys, to return a slice
// of EthRctTrie nodes.
func (rt *rctTrie) GetNodes() ([]*EthRctTrie, error) {
keys, err := rt.getKeys()
if err != nil {
return nil, err
}
var out []*EthRctTrie
for _, k := range keys {
n, err := rt.getNodeFromDB(k)
if err != nil {
return nil, err
}
out = append(out, n)
}
return out, nil
}
// GetLeafNodes invokes the localTrie, which returns a slice
// of EthRctTrie leaf nodes.
func (rt *rctTrie) GetLeafNodes() ([]*EthRctTrie, []*nodeKey, error) {
keys, err := rt.getLeafKeys()
if err != nil {
return nil, nil, err
}
out := make([]*EthRctTrie, 0, len(keys))
for _, k := range keys {
n, err := rt.getNodeFromDB(k.dbKey)
if err != nil {
return nil, nil, err
}
out = append(out, n)
}
return out, keys, nil
}
func (rt *rctTrie) getNodeFromDB(key []byte) (*EthRctTrie, error) {
rawdata, err := rt.db.Get(key)
if err != nil {
return nil, err
}
tn := &TrieNode{
cid: keccak256ToCid(MEthTxReceiptTrie, key),
rawdata: rawdata,
}
return &EthRctTrie{TrieNode: tn}, nil
}

126
statediff/indexer/ipfs/ipld/eth_state.go Normal file
View File

@ -0,0 +1,126 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipld
import (
"fmt"
"io"
"io/ioutil"
"github.com/ipfs/go-cid"
node "github.com/ipfs/go-ipld-format"
"github.com/multiformats/go-multihash"
"github.com/ethereum/go-ethereum/rlp"
)
// EthStateTrie (eth-state-trie, codec 0x96), represents
// a node from the satte trie in ethereum.
type EthStateTrie struct {
*TrieNode
}
// Static (compile time) check that EthStateTrie satisfies the node.Node interface.
var _ node.Node = (*EthStateTrie)(nil)
/*
INPUT
*/
// FromStateTrieRLPFile takes the RLP representation of an ethereum
// state trie node to return it as an IPLD node for further processing.
func FromStateTrieRLPFile(r io.Reader) (*EthStateTrie, error) {
raw, err := ioutil.ReadAll(r)
if err != nil {
return nil, err
}
return FromStateTrieRLP(raw)
}
// FromStateTrieRLP takes the RLP representation of an ethereum
// state trie node to return it as an IPLD node for further processing.
func FromStateTrieRLP(raw []byte) (*EthStateTrie, error) {
c, err := RawdataToCid(MEthStateTrie, raw, multihash.KECCAK_256)
if err != nil {
return nil, err
}
// Let's run the whole mile and process the nodeKind and
// its elements, in case somebody would need this function
// to parse an RLP element from the filesystem
return DecodeEthStateTrie(c, raw)
}
/*
OUTPUT
*/
// DecodeEthStateTrie returns an EthStateTrie object from its cid and rawdata.
func DecodeEthStateTrie(c cid.Cid, b []byte) (*EthStateTrie, error) {
tn, err := decodeTrieNode(c, b, decodeEthStateTrieLeaf)
if err != nil {
return nil, err
}
return &EthStateTrie{TrieNode: tn}, nil
}
// decodeEthStateTrieLeaf parses a eth-tx-trie leaf
// from decoded RLP elements
func decodeEthStateTrieLeaf(i []interface{}) ([]interface{}, error) {
var account EthAccount
err := rlp.DecodeBytes(i[1].([]byte), &account)
if err != nil {
return nil, err
}
c, err := RawdataToCid(MEthAccountSnapshot, i[1].([]byte), multihash.KECCAK_256)
if err != nil {
return nil, err
}
return []interface{}{
i[0].([]byte),
&EthAccountSnapshot{
EthAccount: &account,
cid: c,
rawdata: i[1].([]byte),
},
}, nil
}
/*
Block INTERFACE
*/
// RawData returns the binary of the RLP encode of the state trie node.
func (st *EthStateTrie) RawData() []byte {
return st.rawdata
}
// Cid returns the cid of the state trie node.
func (st *EthStateTrie) Cid() cid.Cid {
return st.cid
}
// String is a helper for output
func (st *EthStateTrie) String() string {
return fmt.Sprintf("<EthereumStateTrie %s>", st.cid)
}
// Loggable returns in a map the type of IPLD Link.
func (st *EthStateTrie) Loggable() map[string]interface{} {
return map[string]interface{}{
"type": "eth-state-trie",
}
}

326
statediff/indexer/ipfs/ipld/eth_state_test.go Normal file
View File

@ -0,0 +1,326 @@
package ipld
import (
"fmt"
"os"
"testing"
"github.com/ipfs/go-cid"
node "github.com/ipfs/go-ipld-format"
)
/*
INPUT
OUTPUT
*/
func TestStateTrieNodeEvenExtensionParsing(t *testing.T) {
fi, err := os.Open("test_data/eth-state-trie-rlp-eb2f5f")
checkError(err, t)
output, err := FromStateTrieRLPFile(fi)
checkError(err, t)
if output.nodeKind != "extension" {
t.Fatalf("Wrong nodeKind\r\nexpected %s\r\ngot %s", "extension", output.nodeKind)
}
if len(output.elements) != 2 {
t.Fatalf("Wrong number of elements for an extension node\r\nexpected %d\r\ngot %d", 2, len(output.elements))
}
if fmt.Sprintf("%x", output.elements[0]) != "0d08" {
t.Fatalf("Wrong key\r\nexpected %s\r\ngot %s", "0d08", fmt.Sprintf("%x", output.elements[0]))
}
if output.elements[1].(cid.Cid).String() !=
"baglacgzalnzmhhnxudxtga6t3do2rctb6ycgyj6mjnycoamlnc733nnbkd6q" {
t.Fatalf("Wrong CID\r\nexpected %s\r\ngot %s", "baglacgzalnzmhhnxudxtga6t3do2rctb6ycgyj6mjnycoamlnc733nnbkd6q", output.elements[1].(cid.Cid).String())
}
}
func TestStateTrieNodeOddExtensionParsing(t *testing.T) {
fi, err := os.Open("test_data/eth-state-trie-rlp-56864f")
checkError(err, t)
output, err := FromStateTrieRLPFile(fi)
checkError(err, t)
if output.nodeKind != "extension" {
t.Fatalf("Wrong nodeKind\r\nexpected %s\r\ngot %s", "extension", output.nodeKind)
}
if len(output.elements) != 2 {
t.Fatalf("Wrong number of elements for an extension node\r\nexpected %d\r\ngot %d", 2, len(output.elements))
}
if fmt.Sprintf("%x", output.elements[0]) != "02" {
t.Fatalf("Wrong key\r\nexpected %s\r\ngot %s", "02", fmt.Sprintf("%x", output.elements[0]))
}
if output.elements[1].(cid.Cid).String() !=
"baglacgzaizf2czb7wztoox4lu23qkwkbfamqsdzcmejzr3rsszrvkaktpfeq" {
t.Fatalf("Wrong CID\r\nexpected %s\r\ngot %s", "baglacgzaizf2czb7wztoox4lu23qkwkbfamqsdzcmejzr3rsszrvkaktpfeq", output.elements[1].(cid.Cid).String())
}
}
func TestStateTrieNodeEvenLeafParsing(t *testing.T) {
fi, err := os.Open("test_data/eth-state-trie-rlp-0e8b34")
checkError(err, t)
output, err := FromStateTrieRLPFile(fi)
checkError(err, t)
if output.nodeKind != "leaf" {
t.Fatalf("Wrong nodeKind\r\nexpected %s\r\ngot %s", "leaf", output.nodeKind)
}
if len(output.elements) != 2 {
t.Fatalf("Wrong number of elements for an extension node\r\nexpected %d\r\ngot %d", 2, len(output.elements))
}
// bd66f60e5b954e1af93ded1b02cb575ff0ed6d9241797eff7576b0bf0637
if fmt.Sprintf("%x", output.elements[0].([]byte)[0:10]) != "0b0d06060f06000e050b" {
t.Fatalf("Wrong key\r\nexpected %s\r\ngot %s", "0b0d06060f06000e050b", fmt.Sprintf("%x", output.elements[0].([]byte)[0:10]))
}
if output.elements[1].(*EthAccountSnapshot).String() !=
"<EthereumAccountSnapshot baglqcgzaf5tapdf2fwb6mo4ijtovqpoi4n3f4jv2yx6avvz6sjypp6vytfva>" {
t.Fatalf("Wrong String()\r\nexpected %s\r\ngot %s", "<EthereumAccountSnapshot baglqcgzaf5tapdf2fwb6mo4ijtovqpoi4n3f4jv2yx6avvz6sjypp6vytfva>", output.elements[1].(*EthAccountSnapshot).String())
}
}
func TestStateTrieNodeOddLeafParsing(t *testing.T) {
fi, err := os.Open("test_data/eth-state-trie-rlp-c9070d")
checkError(err, t)
output, err := FromStateTrieRLPFile(fi)
checkError(err, t)
if output.nodeKind != "leaf" {
t.Fatalf("Wrong nodeKind\r\nexpected %s\r\ngot %s", "leaf", output.nodeKind)
}
if len(output.elements) != 2 {
t.Fatalf("Wrong number of elements for an extension node\r\nexpected %d\r\ngot %d", 2, len(output.elements))
}
// 6c9db9bb545a03425e300f3ee72bae098110336dd3eaf48c20a2e5b6865fc
if fmt.Sprintf("%x", output.elements[0].([]byte)[0:10]) != "060c090d0b090b0b0504" {
t.Fatalf("Wrong key\r\nexpected %s\r\ngot %s", "060c090d0b090b0b0504", fmt.Sprintf("%x", output.elements[0].([]byte)[0:10]))
}
if output.elements[1].(*EthAccountSnapshot).String() !=
"<EthereumAccountSnapshot baglqcgzasckx2alxk43cksshnztjvhfyvbbh6bkp376gtcndm5cg4fkrkhsa>" {
t.Fatalf("Wrong String()\r\nexpected %s\r\ngot %s", "<EthereumAccountSnapshot baglqcgzasckx2alxk43cksshnztjvhfyvbbh6bkp376gtcndm5cg4fkrkhsa>", output.elements[1].(*EthAccountSnapshot).String())
}
}
/*
Block INTERFACE
*/
func TestStateTrieBlockElements(t *testing.T) {
fi, err := os.Open("test_data/eth-state-trie-rlp-d7f897")
checkError(err, t)
output, err := FromStateTrieRLPFile(fi)
checkError(err, t)
if fmt.Sprintf("%x", output.RawData())[:10] != "f90211a090" {
t.Fatalf("Wrong Data\r\nexpected %s\r\ngot %s", "f90211a090", fmt.Sprintf("%x", output.RawData())[:10])
}
if output.Cid().String() !=
"baglacgza274jot5vvr4ntlajtonnkaml5xbm4cts3liye6qxbhndawapavca" {
t.Fatalf("Wrong Cid\r\nexpected %s\r\ngot %s", "baglacgza274jot5vvr4ntlajtonnkaml5xbm4cts3liye6qxbhndawapavca", output.Cid().String())
}
}
func TestStateTrieString(t *testing.T) {
fi, err := os.Open("test_data/eth-state-trie-rlp-d7f897")
checkError(err, t)
output, err := FromStateTrieRLPFile(fi)
checkError(err, t)
if output.String() !=
"<EthereumStateTrie baglacgza274jot5vvr4ntlajtonnkaml5xbm4cts3liye6qxbhndawapavca>" {
t.Fatalf("Wrong String()\r\nexpected %s\r\ngot %s", "<EthereumStateTrie baglacgza274jot5vvr4ntlajtonnkaml5xbm4cts3liye6qxbhndawapavca>", output.String())
}
}
func TestStateTrieLoggable(t *testing.T) {
fi, err := os.Open("test_data/eth-state-trie-rlp-d7f897")
checkError(err, t)
output, err := FromStateTrieRLPFile(fi)
checkError(err, t)
l := output.Loggable()
if _, ok := l["type"]; !ok {
t.Fatal("Loggable map expected the field 'type'")
}
if l["type"] != "eth-state-trie" {
t.Fatalf("Wrong Loggable 'type' value\r\nexpected %s\r\ngot %s", "eth-state-trie", l["type"])
}
}
/*
TRIE NODE (Through EthStateTrie)
Node INTERFACE
*/
func TestTraverseStateTrieWithResolve(t *testing.T) {
var err error
stMap := prepareStateTrieMap(t)
// This is the cid of the root of the block 0
// baglacgza274jot5vvr4ntlajtonnkaml5xbm4cts3liye6qxbhndawapavca
currentNode := stMap["baglacgza274jot5vvr4ntlajtonnkaml5xbm4cts3liye6qxbhndawapavca"]
// This is the path we want to traverse
// The eth address is 0x5abfec25f74cd88437631a7731906932776356f9
// Its keccak-256 is cdd3e25edec0a536a05f5e5ab90a5603624c0ed77453b2e8f955cf8b43d4d0fb
// We use the keccak-256(addr) to traverse the state trie in ethereum.
var traversePath []string
for _, s := range "cdd3e25edec0a536a05f5e5ab90a5603624c0ed77453b2e8f955cf8b43d4d0fb" {
traversePath = append(traversePath, string(s))
}
traversePath = append(traversePath, "balance")
var obj interface{}
for {
obj, traversePath, err = currentNode.Resolve(traversePath)
link, ok := obj.(*node.Link)
if !ok {
break
}
if err != nil {
t.Fatal("Error should be nil")
}
currentNode = stMap[link.Cid.String()]
if currentNode == nil {
t.Fatal("state trie node not found in memory map")
}
}
if fmt.Sprintf("%v", obj) != "11901484239480000000000000" {
t.Fatalf("Wrong balance value\r\nexpected %s\r\ngot %s", "11901484239480000000000000", fmt.Sprintf("%v", obj))
}
}
func TestStateTrieResolveLinks(t *testing.T) {
fi, err := os.Open("test_data/eth-state-trie-rlp-eb2f5f")
checkError(err, t)
stNode, err := FromStateTrieRLPFile(fi)
checkError(err, t)
// bad case
obj, rest, err := stNode.ResolveLink([]string{"supercalifragilist"})
if obj != nil {
t.Fatalf("Expected obj to be nil")
}
if rest != nil {
t.Fatal("Expected rest to be nil")
}
if err.Error() != "invalid path element" {
t.Fatalf("Wrong error\r\nexpected %s\r\ngot %s", "invalid path element", err.Error())
}
// good case
obj, rest, err = stNode.ResolveLink([]string{"d8"})
if obj == nil {
t.Fatalf("Expected a not nil obj to be returned")
}
if rest != nil {
t.Fatal("Expected rest to be nil")
}
if err != nil {
t.Fatal("Expected error to be nil")
}
}
func TestStateTrieCopy(t *testing.T) {
fi, err := os.Open("test_data/eth-state-trie-rlp-eb2f5f")
checkError(err, t)
stNode, err := FromStateTrieRLPFile(fi)
checkError(err, t)
defer func() {
r := recover()
if r == nil {
t.Fatal("Expected panic")
}
if r != "implement me" {
t.Fatalf("Wrong panic message\r\nexpected %s\r\ngot %s", "'implement me'", r)
}
}()
_ = stNode.Copy()
}
func TestStateTrieStat(t *testing.T) {
fi, err := os.Open("test_data/eth-state-trie-rlp-eb2f5f")
checkError(err, t)
stNode, err := FromStateTrieRLPFile(fi)
checkError(err, t)
obj, err := stNode.Stat()
if obj == nil {
t.Fatal("Expected a not null object node.NodeStat")
}
if err != nil {
t.Fatal("Expected a nil error")
}
}
func TestStateTrieSize(t *testing.T) {
fi, err := os.Open("test_data/eth-state-trie-rlp-eb2f5f")
checkError(err, t)
stNode, err := FromStateTrieRLPFile(fi)
checkError(err, t)
size, err := stNode.Size()
if size != uint64(0) {
t.Fatalf("Wrong size\r\nexpected %d\r\ngot %d", 0, size)
}
if err != nil {
t.Fatal("Expected a nil error")
}
}
func prepareStateTrieMap(t *testing.T) map[string]*EthStateTrie {
filepaths := []string{
"test_data/eth-state-trie-rlp-0e8b34",
"test_data/eth-state-trie-rlp-56864f",
"test_data/eth-state-trie-rlp-6fc2d7",
"test_data/eth-state-trie-rlp-727994",
"test_data/eth-state-trie-rlp-c9070d",
"test_data/eth-state-trie-rlp-d5be90",
"test_data/eth-state-trie-rlp-d7f897",
"test_data/eth-state-trie-rlp-eb2f5f",
}
out := make(map[string]*EthStateTrie)
for _, fp := range filepaths {
fi, err := os.Open(fp)
checkError(err, t)
stateTrieNode, err := FromStateTrieRLPFile(fi)
checkError(err, t)
out[stateTrieNode.Cid().String()] = stateTrieNode
}
return out
}

112
statediff/indexer/ipfs/ipld/eth_storage.go Normal file
View File

@ -0,0 +1,112 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipld
import (
"fmt"
"io"
"io/ioutil"
"github.com/ipfs/go-cid"
node "github.com/ipfs/go-ipld-format"
"github.com/multiformats/go-multihash"
)
// EthStorageTrie (eth-storage-trie, codec 0x98), represents
// a node from the storage trie in ethereum.
type EthStorageTrie struct {
*TrieNode
}
// Static (compile time) check that EthStorageTrie satisfies the node.Node interface.
var _ node.Node = (*EthStorageTrie)(nil)
/*
INPUT
*/
// FromStorageTrieRLPFile takes the RLP representation of an ethereum
// storage trie node to return it as an IPLD node for further processing.
func FromStorageTrieRLPFile(r io.Reader) (*EthStorageTrie, error) {
raw, err := ioutil.ReadAll(r)
if err != nil {
return nil, err
}
return FromStorageTrieRLP(raw)
}
// FromStorageTrieRLP takes the RLP representation of an ethereum
// storage trie node to return it as an IPLD node for further processing.
func FromStorageTrieRLP(raw []byte) (*EthStorageTrie, error) {
c, err := RawdataToCid(MEthStorageTrie, raw, multihash.KECCAK_256)
if err != nil {
return nil, err
}
// Let's run the whole mile and process the nodeKind and
// its elements, in case somebody would need this function
// to parse an RLP element from the filesystem
return DecodeEthStorageTrie(c, raw)
}
/*
OUTPUT
*/
// DecodeEthStorageTrie returns an EthStorageTrie object from its cid and rawdata.
func DecodeEthStorageTrie(c cid.Cid, b []byte) (*EthStorageTrie, error) {
tn, err := decodeTrieNode(c, b, decodeEthStorageTrieLeaf)
if err != nil {
return nil, err
}
return &EthStorageTrie{TrieNode: tn}, nil
}
// decodeEthStorageTrieLeaf parses a eth-tx-trie leaf
// from decoded RLP elements
func decodeEthStorageTrieLeaf(i []interface{}) ([]interface{}, error) {
return []interface{}{
i[0].([]byte),
i[1].([]byte),
}, nil
}
/*
Block INTERFACE
*/
// RawData returns the binary of the RLP encode of the storage trie node.
func (st *EthStorageTrie) RawData() []byte {
return st.rawdata
}
// Cid returns the cid of the storage trie node.
func (st *EthStorageTrie) Cid() cid.Cid {
return st.cid
}
// String is a helper for output
func (st *EthStorageTrie) String() string {
return fmt.Sprintf("<EthereumStorageTrie %s>", st.cid)
}
// Loggable returns in a map the type of IPLD Link.
func (st *EthStorageTrie) Loggable() map[string]interface{} {
return map[string]interface{}{
"type": "eth-storage-trie",
}
}

140
statediff/indexer/ipfs/ipld/eth_storage_test.go Normal file
View File

@ -0,0 +1,140 @@
package ipld
import (
"fmt"
"os"
"testing"
"github.com/ipfs/go-cid"
)
/*
INPUT
OUTPUT
*/
func TestStorageTrieNodeExtensionParsing(t *testing.T) {
fi, err := os.Open("test_data/eth-storage-trie-rlp-113049")
checkError(err, t)
output, err := FromStateTrieRLPFile(fi)
checkError(err, t)
if output.nodeKind != "extension" {
t.Fatalf("Wrong nodeKind\r\nexpected %s\r\ngot %s", "extension", output.nodeKind)
}
if len(output.elements) != 2 {
t.Fatalf("Wrong number of elements for an extension node\r\nexpected %d\r\ngot %d", 2, len(output.elements))
}
if fmt.Sprintf("%x", output.elements[0]) != "0a" {
t.Fatalf("Wrong key\r\nexpected %s\r\ngot %s", "0a", fmt.Sprintf("%x", output.elements[0]))
}
if output.elements[1].(cid.Cid).String() !=
"baglacgzautxeutufae7owyrezfvwpan2vusocmxgzwqhzrhjbwprp2texgsq" {
t.Fatalf("Wrong CID\r\nexpected %s\r\ngot %s", "baglacgzautxeutufae7owyrezfvwpan2vusocmxgzwqhzrhjbwprp2texgsq", output.elements[1].(cid.Cid).String())
}
}
func TestStateTrieNodeLeafParsing(t *testing.T) {
fi, err := os.Open("test_data/eth-storage-trie-rlp-ffbcad")
checkError(err, t)
output, err := FromStorageTrieRLPFile(fi)
checkError(err, t)
if output.nodeKind != "leaf" {
t.Fatalf("Wrong nodeKind\r\nexpected %s\r\ngot %s", "leaf", output.nodeKind)
}
if len(output.elements) != 2 {
t.Fatalf("Wrong number of elements for an leaf node\r\nexpected %d\r\ngot %d", 2, len(output.elements))
}
// 2ee1ae9c502e48e0ed528b7b39ac569cef69d7844b5606841a7f3fe898a2
if fmt.Sprintf("%x", output.elements[0].([]byte)[:10]) != "020e0e010a0e090c0500" {
t.Fatalf("Wrong key\r\nexpected %s\r\ngot %s", "020e0e010a0e090c0500", fmt.Sprintf("%x", output.elements[0].([]byte)[:10]))
}
if fmt.Sprintf("%x", output.elements[1]) != "89056c31f304b2530000" {
t.Fatalf("Wrong Value\r\nexpected %s\r\ngot %s", "89056c31f304b2530000", fmt.Sprintf("%x", output.elements[1]))
}
}
func TestStateTrieNodeBranchParsing(t *testing.T) {
fi, err := os.Open("test_data/eth-storage-trie-rlp-ffc25c")
checkError(err, t)
output, err := FromStateTrieRLPFile(fi)
checkError(err, t)
if output.nodeKind != "branch" {
t.Fatalf("Wrong nodeKind\r\nexpected %s\r\ngot %s", "branch", output.nodeKind)
}
if len(output.elements) != 17 {
t.Fatalf("Wrong number of elements for an branch node\r\nexpected %d\r\ngot %d", 17, len(output.elements))
}
if fmt.Sprintf("%s", output.elements[4]) !=
"baglacgzadqhbmlxrxtw5hplcq5jn74p4dceryzw664w3237ra52dnghbjpva" {
t.Fatalf("Wrong Cid\r\nexpected %s\r\ngot %s", "baglacgzadqhbmlxrxtw5hplcq5jn74p4dceryzw664w3237ra52dnghbjpva", fmt.Sprintf("%s", output.elements[4]))
}
if fmt.Sprintf("%s", output.elements[10]) !=
"baglacgza77d37i2v6uhtzeeq4vngragjbgbwq3lylpoc3lihenvzimybzxmq" {
t.Fatalf("Wrong Cid\r\nexpected %s\r\ngot %s", "baglacgza77d37i2v6uhtzeeq4vngragjbgbwq3lylpoc3lihenvzimybzxmq", fmt.Sprintf("%s", output.elements[10]))
}
}
/*
Block INTERFACE
*/
func TestStorageTrieBlockElements(t *testing.T) {
fi, err := os.Open("test_data/eth-storage-trie-rlp-ffbcad")
checkError(err, t)
output, err := FromStorageTrieRLPFile(fi)
checkError(err, t)
if fmt.Sprintf("%x", output.RawData())[:10] != "eb9f202ee1" {
t.Fatalf("Wrong Data\r\nexpected %s\r\ngot %s", "eb9f202ee1", fmt.Sprintf("%x", output.RawData())[:10])
}
if output.Cid().String() !=
"bagmacgza766k3oprj2qxn36eycw55pogmu3dwtfay6zdh6ajrhvw3b2nqg5a" {
t.Fatalf("Wrong Cid\r\nexpected %s\r\ngot %s", "bagmacgza766k3oprj2qxn36eycw55pogmu3dwtfay6zdh6ajrhvw3b2nqg5a", output.Cid().String())
}
}
func TestStorageTrieString(t *testing.T) {
fi, err := os.Open("test_data/eth-storage-trie-rlp-ffbcad")
checkError(err, t)
output, err := FromStorageTrieRLPFile(fi)
checkError(err, t)
if output.String() !=
"<EthereumStorageTrie bagmacgza766k3oprj2qxn36eycw55pogmu3dwtfay6zdh6ajrhvw3b2nqg5a>" {
t.Fatalf("Wrong String()\r\nexpected %s\r\ngot %s", "<EthereumStorageTrie bagmacgza766k3oprj2qxn36eycw55pogmu3dwtfay6zdh6ajrhvw3b2nqg5a>", output.String())
}
}
func TestStorageTrieLoggable(t *testing.T) {
fi, err := os.Open("test_data/eth-storage-trie-rlp-ffbcad")
checkError(err, t)
output, err := FromStorageTrieRLPFile(fi)
checkError(err, t)
l := output.Loggable()
if _, ok := l["type"]; !ok {
t.Fatal("Loggable map expected the field 'type'")
}
if l["type"] != "eth-storage-trie" {
t.Fatalf("Wrong Loggable 'type' value\r\nexpected %s\r\ngot %s", "eth-storage-trie", l["type"])
}
}

237
statediff/indexer/ipfs/ipld/eth_tx.go Normal file
View File

@ -0,0 +1,237 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipld
import (
"encoding/json"
"fmt"
"strconv"
"strings"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ipfs/go-cid"
node "github.com/ipfs/go-ipld-format"
mh "github.com/multiformats/go-multihash"
)
// EthTx (eth-tx codec 0x93) represents an ethereum transaction
type EthTx struct {
*types.Transaction
cid cid.Cid
rawdata []byte
}
// Static (compile time) check that EthTx satisfies the node.Node interface.
var _ node.Node = (*EthTx)(nil)
/*
INPUT
*/
// NewEthTx converts a *types.Transaction to an EthTx IPLD node
func NewEthTx(tx *types.Transaction) (*EthTx, error) {
txRaw, err := tx.MarshalBinary()
if err != nil {
return nil, err
}
c, err := RawdataToCid(MEthTx, txRaw, mh.KECCAK_256)
if err != nil {
return nil, err
}
return &EthTx{
Transaction: tx,
cid: c,
rawdata: txRaw,
}, nil
}
/*
OUTPUT
*/
// DecodeEthTx takes a cid and its raw binary data
// from IPFS and returns an EthTx object for further processing.
func DecodeEthTx(c cid.Cid, b []byte) (*EthTx, error) {
t := new(types.Transaction)
if err := t.UnmarshalBinary(b); err != nil {
return nil, err
}
return &EthTx{
Transaction: t,
cid: c,
rawdata: b,
}, nil
}
/*
Block INTERFACE
*/
// RawData returns the binary of the RLP encode of the transaction.
func (t *EthTx) RawData() []byte {
return t.rawdata
}
// Cid returns the cid of the transaction.
func (t *EthTx) Cid() cid.Cid {
return t.cid
}
// String is a helper for output
func (t *EthTx) String() string {
return fmt.Sprintf("<EthereumTx %s>", t.cid)
}
// Loggable returns in a map the type of IPLD Link.
func (t *EthTx) Loggable() map[string]interface{} {
return map[string]interface{}{
"type": "eth-tx",
}
}
/*
Node INTERFACE
*/
// Resolve resolves a path through this node, stopping at any link boundary
// and returning the object found as well as the remaining path to traverse
func (t *EthTx) Resolve(p []string) (interface{}, []string, error) {
if len(p) == 0 {
return t, nil, nil
}
if len(p) > 1 {
return nil, nil, fmt.Errorf("unexpected path elements past %s", p[0])
}
switch p[0] {
case "type":
return t.Type(), nil, nil
case "gas":
return t.Gas(), nil, nil
case "gasPrice":
return t.GasPrice(), nil, nil
case "input":
return fmt.Sprintf("%x", t.Data()), nil, nil
case "nonce":
return t.Nonce(), nil, nil
case "r":
_, r, _ := t.RawSignatureValues()
return hexutil.EncodeBig(r), nil, nil
case "s":
_, _, s := t.RawSignatureValues()
return hexutil.EncodeBig(s), nil, nil
case "toAddress":
return t.To(), nil, nil
case "v":
v, _, _ := t.RawSignatureValues()
return hexutil.EncodeBig(v), nil, nil
case "value":
return hexutil.EncodeBig(t.Value()), nil, nil
default:
return nil, nil, ErrInvalidLink
}
}
// Tree lists all paths within the object under 'path', and up to the given depth.
// To list the entire object (similar to `find .`) pass "" and -1
func (t *EthTx) Tree(p string, depth int) []string {
if p != "" || depth == 0 {
return nil
}
return []string{"type", "gas", "gasPrice", "input", "nonce", "r", "s", "toAddress", "v", "value"}
}
// ResolveLink is a helper function that calls resolve and asserts the
// output is a link
func (t *EthTx) ResolveLink(p []string) (*node.Link, []string, error) {
obj, rest, err := t.Resolve(p)
if err != nil {
return nil, nil, err
}
if lnk, ok := obj.(*node.Link); ok {
return lnk, rest, nil
}
return nil, nil, fmt.Errorf("resolved item was not a link")
}
// Copy will go away. It is here to comply with the interface.
func (t *EthTx) Copy() node.Node {
panic("implement me")
}
// Links is a helper function that returns all links within this object
func (t *EthTx) Links() []*node.Link {
return nil
}
// Stat will go away. It is here to comply with the interface.
func (t *EthTx) Stat() (*node.NodeStat, error) {
return &node.NodeStat{}, nil
}
// Size will go away. It is here to comply with the interface. It returns the byte size for the transaction
func (t *EthTx) Size() (uint64, error) {
spl := strings.Split(t.Transaction.Size().String(), " ")
size, units := spl[0], spl[1]
floatSize, err := strconv.ParseFloat(size, 64)
if err != nil {
return 0, err
}
var byteSize uint64
switch units {
case "B":
byteSize = uint64(floatSize)
case "KB":
byteSize = uint64(floatSize * 1000)
case "MB":
byteSize = uint64(floatSize * 1000000)
case "GB":
byteSize = uint64(floatSize * 1000000000)
case "TB":
byteSize = uint64(floatSize * 1000000000000)
default:
return 0, fmt.Errorf("unreconginized units %s", units)
}
return byteSize, nil
}
/*
EthTx functions
*/
// MarshalJSON processes the transaction into readable JSON format.
func (t *EthTx) MarshalJSON() ([]byte, error) {
v, r, s := t.RawSignatureValues()
out := map[string]interface{}{
"gas": t.Gas(),
"gasPrice": hexutil.EncodeBig(t.GasPrice()),
"input": fmt.Sprintf("%x", t.Data()),
"nonce": t.Nonce(),
"r": hexutil.EncodeBig(r),
"s": hexutil.EncodeBig(s),
"toAddress": t.To(),
"v": hexutil.EncodeBig(v),
"value": hexutil.EncodeBig(t.Value()),
}
return json.Marshal(out)
}

412
statediff/indexer/ipfs/ipld/eth_tx_test.go Normal file
View File

@ -0,0 +1,412 @@
package ipld
import (
"encoding/hex"
"fmt"
"os"
"strconv"
"strings"
"testing"
block "github.com/ipfs/go-block-format"
"github.com/multiformats/go-multihash"
)
/*
EthBlock
INPUT
*/
func TestTxInBlockBodyRlpParsing(t *testing.T) {
fi, err := os.Open("test_data/eth-block-body-rlp-999999")
checkError(err, t)
_, output, _, err := FromBlockRLP(fi)
checkError(err, t)
if len(output) != 11 {
t.Fatalf("Wrong number of parsed txs\r\nexpected %d\r\ngot %d", 11, len(output))
}
// Oh, let's just grab the last element and one from the middle
testTx05Fields(output[5], t)
testTx10Fields(output[10], t)
}
func TestTxInBlockHeaderRlpParsing(t *testing.T) {
fi, err := os.Open("test_data/eth-block-header-rlp-999999")
checkError(err, t)
_, output, _, err := FromBlockRLP(fi)
checkError(err, t)
if len(output) != 0 {
t.Fatalf("Wrong number of txs\r\nexpected %d\r\ngot %d", 0, len(output))
}
}
func TestTxInBlockBodyJsonParsing(t *testing.T) {
fi, err := os.Open("test_data/eth-block-body-json-999999")
checkError(err, t)
_, output, _, err := FromBlockJSON(fi)
checkError(err, t)
if len(output) != 11 {
t.Fatalf("Wrong number of parsed txs\r\nexpected %d\r\ngot %d", 11, len(output))
}
testTx05Fields(output[5], t)
testTx10Fields(output[10], t)
}
/*
OUTPUT
*/
func TestDecodeTransaction(t *testing.T) {
// Prepare the "fetched transaction".
// This one is supposed to be in the datastore already,
// and given away by github.com/ipfs/go-ipfs/merkledag
rawTransactionString :=
"f86c34850df84758008252089432be343b94f860124dc4fee278fdcbd38c102d88880f25" +
"8512af0d4000801ba0e9a25c929c26d1a95232ba75aef419a91b470651eb77614695e16c" +
"5ba023e383a0679fb2fc0d0b0f3549967c0894ee7d947f07d238a83ef745bc3ced5143a4af36"
rawTransaction, err := hex.DecodeString(rawTransactionString)
checkError(err, t)
c, err := RawdataToCid(MEthTx, rawTransaction, multihash.KECCAK_256)
checkError(err, t)
// Just to clarify: This `block` is an IPFS block
storedTransaction, err := block.NewBlockWithCid(rawTransaction, c)
checkError(err, t)
// Now the proper test
ethTransaction, err := DecodeEthTx(storedTransaction.Cid(), storedTransaction.RawData())
checkError(err, t)
testTx05Fields(ethTransaction, t)
}
/*
Block INTERFACE
*/
func TestEthTxLoggable(t *testing.T) {
txs := prepareParsedTxs(t)
l := txs[0].Loggable()
if _, ok := l["type"]; !ok {
t.Fatal("Loggable map expected the field 'type'")
}
if l["type"] != "eth-tx" {
t.Fatalf("Wrong Loggable 'type' value\r\nexpected %s\r\ngot %s", "eth-tx", l["type"])
}
}
/*
Node INTERFACE
*/
func TestEthTxResolve(t *testing.T) {
tx := prepareParsedTxs(t)[0]
// Empty path
obj, rest, err := tx.Resolve([]string{})
rtx, ok := obj.(*EthTx)
if !ok {
t.Fatal("Wrong type of returned object")
}
if rtx.Cid() != tx.Cid() {
t.Fatalf("Wrong CID\r\nexpected %s\r\ngot %s", tx.Cid().String(), rtx.Cid().String())
}
if rest != nil {
t.Fatal("est should be nil")
}
if err != nil {
t.Fatal("err should be nil")
}
// len(p) > 1
badCases := [][]string{
{"two", "elements"},
{"here", "three", "elements"},
{"and", "here", "four", "elements"},
}
for _, bc := range badCases {
obj, rest, err = tx.Resolve(bc)
if obj != nil {
t.Fatal("obj should be nil")
}
if rest != nil {
t.Fatal("rest should be nil")
}
if err.Error() != fmt.Sprintf("unexpected path elements past %s", bc[0]) {
t.Fatalf("wrong error\r\nexpected %s\r\ngot %s", fmt.Sprintf("unexpected path elements past %s", bc[0]), err.Error())
}
}
moreBadCases := []string{
"i",
"am",
"not",
"a",
"tx",
"field",
}
for _, mbc := range moreBadCases {
obj, rest, err = tx.Resolve([]string{mbc})
if obj != nil {
t.Fatal("obj should be nil")
}
if rest != nil {
t.Fatal("rest should be nil")
}
if err != ErrInvalidLink {
t.Fatalf("wrong error\r\nexpected %s\r\ngot %s", ErrInvalidLink, err)
}
}
goodCases := []string{
"gas",
"gasPrice",
"input",
"nonce",
"r",
"s",
"toAddress",
"v",
"value",
}
for _, gc := range goodCases {
_, _, err = tx.Resolve([]string{gc})
if err != nil {
t.Fatalf("error should be nil %v", gc)
}
}
}
func TestEthTxTree(t *testing.T) {
tx := prepareParsedTxs(t)[0]
_ = tx
// Bad cases
tree := tx.Tree("non-empty-string", 0)
if tree != nil {
t.Fatal("Expected nil to be returned")
}
tree = tx.Tree("non-empty-string", 1)
if tree != nil {
t.Fatal("Expected nil to be returned")
}
tree = tx.Tree("", 0)
if tree != nil {
t.Fatal("Expected nil to be returned")
}
// Good cases
tree = tx.Tree("", 1)
lookupElements := map[string]interface{}{
"type": nil,
"gas": nil,
"gasPrice": nil,
"input": nil,
"nonce": nil,
"r": nil,
"s": nil,
"toAddress": nil,
"v": nil,
"value": nil,
}
if len(tree) != len(lookupElements) {
t.Fatalf("Wrong number of elements\r\nexpected %d\r\ngot %d", len(lookupElements), len(tree))
}
for _, te := range tree {
if _, ok := lookupElements[te]; !ok {
t.Fatalf("Unexpected Element: %v", te)
}
}
}
func TestEthTxResolveLink(t *testing.T) {
tx := prepareParsedTxs(t)[0]
// bad case
obj, rest, err := tx.ResolveLink([]string{"supercalifragilist"})
if obj != nil {
t.Fatalf("Expected obj to be nil")
}
if rest != nil {
t.Fatal("Expected rest to be nil")
}
if err != ErrInvalidLink {
t.Fatalf("Wrong error\r\nexpected %s\r\ngot %s", ErrInvalidLink, err.Error())
}
// good case
obj, rest, err = tx.ResolveLink([]string{"nonce"})
if obj != nil {
t.Fatalf("Expected obj to be nil")
}
if rest != nil {
t.Fatal("Expected rest to be nil")
}
if err.Error() != "resolved item was not a link" {
t.Fatalf("Wrong error\r\nexpected %s\r\ngot %s", "resolved item was not a link", err.Error())
}
}
func TestEthTxCopy(t *testing.T) {
tx := prepareParsedTxs(t)[0]
defer func() {
r := recover()
if r == nil {
t.Fatal("Expected panic")
}
if r != "implement me" {
t.Fatalf("Wrong panic message\r\nexpected %s\r\ngot %s", "'implement me'", r)
}
}()
_ = tx.Copy()
}
func TestEthTxLinks(t *testing.T) {
tx := prepareParsedTxs(t)[0]
if tx.Links() != nil {
t.Fatal("Links() expected to return nil")
}
}
func TestEthTxStat(t *testing.T) {
tx := prepareParsedTxs(t)[0]
obj, err := tx.Stat()
if obj == nil {
t.Fatal("Expected a not null object node.NodeStat")
}
if err != nil {
t.Fatal("Expected a nil error")
}
}
func TestEthTxSize(t *testing.T) {
tx := prepareParsedTxs(t)[0]
size, err := tx.Size()
checkError(err, t)
spl := strings.Split(tx.Transaction.Size().String(), " ")
expectedSize, units := spl[0], spl[1]
floatSize, err := strconv.ParseFloat(expectedSize, 64)
checkError(err, t)
var byteSize uint64
switch units {
case "B":
byteSize = uint64(floatSize)
case "KB":
byteSize = uint64(floatSize * 1000)
case "MB":
byteSize = uint64(floatSize * 1000000)
case "GB":
byteSize = uint64(floatSize * 1000000000)
case "TB":
byteSize = uint64(floatSize * 1000000000000)
default:
t.Fatal("Unexpected size units")
}
if size != byteSize {
t.Fatalf("Wrong size\r\nexpected %d\r\ngot %d", byteSize, size)
}
}
/*
AUXILIARS
*/
// prepareParsedTxs is a convenienve method
func prepareParsedTxs(t *testing.T) []*EthTx {
fi, err := os.Open("test_data/eth-block-body-rlp-999999")
checkError(err, t)
_, output, _, err := FromBlockRLP(fi)
checkError(err, t)
return output
}
func testTx05Fields(ethTx *EthTx, t *testing.T) {
// Was the cid calculated?
if ethTx.Cid().String() != "bagjqcgzawhfnvdnpmpcfoug7d3tz53k2ht3cidr45pnw3y7snpd46azbpp2a" {
t.Fatalf("Wrong cid\r\nexpected %s\r\ngot %s\r\n", "bagjqcgzawhfnvdnpmpcfoug7d3tz53k2ht3cidr45pnw3y7snpd46azbpp2a", ethTx.Cid().String())
}
// Do we have the rawdata available?
if fmt.Sprintf("%x", ethTx.RawData()[:10]) != "f86c34850df847580082" {
t.Fatalf("Wrong Rawdata\r\nexpected %s\r\ngot %s", "f86c34850df847580082", fmt.Sprintf("%x", ethTx.RawData()[:10]))
}
// Proper Fields of types.Transaction
if fmt.Sprintf("%x", ethTx.To()) != "32be343b94f860124dc4fee278fdcbd38c102d88" {
t.Fatalf("Wrong Recipient\r\nexpected %s\r\ngot %s", "32be343b94f860124dc4fee278fdcbd38c102d88", fmt.Sprintf("%x", ethTx.To()))
}
if len(ethTx.Data()) != 0 {
t.Fatalf("Wrong len of Data\r\nexpected %d\r\ngot %d", 0, len(ethTx.Data()))
}
if fmt.Sprintf("%v", ethTx.Gas()) != "21000" {
t.Fatalf("Wrong Gas\r\nexpected %s\r\ngot %s", "21000", fmt.Sprintf("%v", ethTx.Gas()))
}
if fmt.Sprintf("%v", ethTx.Value()) != "1091424800000000000" {
t.Fatalf("Wrong Value\r\nexpected %s\r\ngot %s", "1091424800000000000", fmt.Sprintf("%v", ethTx.Value()))
}
if fmt.Sprintf("%v", ethTx.Nonce()) != "52" {
t.Fatalf("Wrong Nonce\r\nexpected %s\r\ngot %s", "52", fmt.Sprintf("%v", ethTx.Nonce()))
}
if fmt.Sprintf("%v", ethTx.GasPrice()) != "60000000000" {
t.Fatalf("Wrong Gas Price\r\nexpected %s\r\ngot %s", "60000000000", fmt.Sprintf("%v", ethTx.GasPrice()))
}
}
func testTx10Fields(ethTx *EthTx, t *testing.T) {
// Was the cid calculated?
if ethTx.Cid().String() != "bagjqcgzaykakwayoec6j55zmq62cbvmplgf5u5j67affge3ksi4ermgitjoa" {
t.Fatalf("Wrong Cid\r\nexpected %s\r\ngot %s", "bagjqcgzaykakwayoec6j55zmq62cbvmplgf5u5j67affge3ksi4ermgitjoa", ethTx.Cid().String())
}
// Do we have the rawdata available?
if fmt.Sprintf("%x", ethTx.RawData()[:10]) != "f8708302a120850ba43b" {
t.Fatalf("Wrong Rawdata\r\nexpected %s\r\ngot %s", "f8708302a120850ba43b", fmt.Sprintf("%x", ethTx.RawData()[:10]))
}
// Proper Fields of types.Transaction
if fmt.Sprintf("%x", ethTx.To()) != "1c51bf013add0857c5d9cf2f71a7f15ca93d4816" {
t.Fatalf("Wrong Recipient\r\nexpected %s\r\ngot %s", "1c51bf013add0857c5d9cf2f71a7f15ca93d4816", fmt.Sprintf("%x", ethTx.To()))
}
if len(ethTx.Data()) != 0 {
t.Fatalf("Wrong len of Data\r\nexpected %d\r\ngot %d", 0, len(ethTx.Data()))
}
if fmt.Sprintf("%v", ethTx.Gas()) != "90000" {
t.Fatalf("Wrong Gas\r\nexpected %s\r\ngot %s", "90000", fmt.Sprintf("%v", ethTx.Gas()))
}
if fmt.Sprintf("%v", ethTx.Value()) != "1049756850000000000" {
t.Fatalf("Wrong Value\r\nexpected %s\r\ngot %s", "1049756850000000000", fmt.Sprintf("%v", ethTx.Value()))
}
if fmt.Sprintf("%v", ethTx.Nonce()) != "172320" {
t.Fatalf("Wrong Nonce\r\nexpected %s\r\ngot %s", "172320", fmt.Sprintf("%v", ethTx.Nonce()))
}
if fmt.Sprintf("%v", ethTx.GasPrice()) != "50000000000" {
t.Fatalf("Wrong Gas Price\r\nexpected %s\r\ngot %s", "50000000000", fmt.Sprintf("%v", ethTx.GasPrice()))
}
}

146
statediff/indexer/ipfs/ipld/eth_tx_trie.go Normal file
View File

@ -0,0 +1,146 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipld
import (
"fmt"
"github.com/ipfs/go-cid"
node "github.com/ipfs/go-ipld-format"
"github.com/multiformats/go-multihash"
"github.com/ethereum/go-ethereum/core/types"
)
// EthTxTrie (eth-tx-trie codec 0x92) represents
// a node from the transaction trie in ethereum.
type EthTxTrie struct {
*TrieNode
}
// Static (compile time) check that EthTxTrie satisfies the node.Node interface.
var _ node.Node = (*EthTxTrie)(nil)
/*
INPUT
*/
// To create a proper trie of the eth-tx-trie objects, it is required
// to input all transactions belonging to a forest in a single step.
// We are adding the transactions, and creating its trie on
// block body parsing time.
/*
OUTPUT
*/
// DecodeEthTxTrie returns an EthTxTrie object from its cid and rawdata.
func DecodeEthTxTrie(c cid.Cid, b []byte) (*EthTxTrie, error) {
tn, err := decodeTrieNode(c, b, decodeEthTxTrieLeaf)
if err != nil {
return nil, err
}
return &EthTxTrie{TrieNode: tn}, nil
}
// decodeEthTxTrieLeaf parses a eth-tx-trie leaf
//from decoded RLP elements
func decodeEthTxTrieLeaf(i []interface{}) ([]interface{}, error) {
t := new(types.Transaction)
if err := t.UnmarshalBinary(i[1].([]byte)); err != nil {
return nil, err
}
c, err := RawdataToCid(MEthTx, i[1].([]byte), multihash.KECCAK_256)
if err != nil {
return nil, err
}
return []interface{}{
i[0].([]byte),
&EthTx{
Transaction: t,
cid: c,
rawdata: i[1].([]byte),
},
}, nil
}
/*
Block INTERFACE
*/
// RawData returns the binary of the RLP encode of the transaction.
func (t *EthTxTrie) RawData() []byte {
return t.rawdata
}
// Cid returns the cid of the transaction.
func (t *EthTxTrie) Cid() cid.Cid {
return t.cid
}
// String is a helper for output
func (t *EthTxTrie) String() string {
return fmt.Sprintf("<EthereumTxTrie %s>", t.cid)
}
// Loggable returns in a map the type of IPLD Link.
func (t *EthTxTrie) Loggable() map[string]interface{} {
return map[string]interface{}{
"type": "eth-tx-trie",
}
}
/*
EthTxTrie functions
*/
// txTrie wraps a localTrie for use on the transaction trie.
type txTrie struct {
*localTrie
}
// newTxTrie initializes and returns a txTrie.
func newTxTrie() *txTrie {
return &txTrie{
localTrie: newLocalTrie(),
}
}
// getNodes invokes the localTrie, which computes the root hash of the
// transaction trie and returns its database keys, to return a slice
// of EthTxTrie nodes.
func (tt *txTrie) getNodes() ([]*EthTxTrie, error) {
keys, err := tt.getKeys()
if err != nil {
return nil, err
}
var out []*EthTxTrie
for _, k := range keys {
rawdata, err := tt.db.Get(k)
if err != nil {
return nil, err
}
tn := &TrieNode{
cid: keccak256ToCid(MEthTxTrie, k),
rawdata: rawdata,
}
out = append(out, &EthTxTrie{TrieNode: tn})
}
return out, nil
}

504
statediff/indexer/ipfs/ipld/eth_tx_trie_test.go Normal file
View File

@ -0,0 +1,504 @@
package ipld
import (
"encoding/hex"
"encoding/json"
"fmt"
"os"
"testing"
block "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
node "github.com/ipfs/go-ipld-format"
"github.com/multiformats/go-multihash"
)
/*
EthBlock
*/
func TestTxTriesInBlockBodyJSONParsing(t *testing.T) {
// HINT: 306 txs
// cat test_data/eth-block-body-json-4139497 | jsontool | grep transactionIndex | wc -l
// or, https://etherscan.io/block/4139497
fi, err := os.Open("test_data/eth-block-body-json-4139497")
checkError(err, t)
_, _, output, err := FromBlockJSON(fi)
checkError(err, t)
if len(output) != 331 {
t.Fatalf("Wrong number of obtained tx trie nodes\r\nexpected %d\r\n got %d", 331, len(output))
}
}
/*
OUTPUT
*/
func TestTxTrieDecodeExtension(t *testing.T) {
ethTxTrie := prepareDecodedEthTxTrieExtension(t)
if ethTxTrie.nodeKind != "extension" {
t.Fatalf("Wrong nodeKind\r\nexpected %s\r\ngot %s", "extension", ethTxTrie.nodeKind)
}
if len(ethTxTrie.elements) != 2 {
t.Fatalf("Wrong number of elements for an extension node\r\nexpected %d\r\ngot %d", 2, len(ethTxTrie.elements))
}
if fmt.Sprintf("%x", ethTxTrie.elements[0].([]byte)) != "0001" {
t.Fatalf("Wrong key\r\nexpected %s\r\ngot %s", "0001", fmt.Sprintf("%x", ethTxTrie.elements[0].([]byte)))
}
if ethTxTrie.elements[1].(cid.Cid).String() !=
"bagjacgzak6wdjvshdtb7lrvlteweyd7f5qjr3dmzmh7g2xpi4xrwoujsio2a" {
t.Fatalf("Wrong CID\r\nexpected %s\r\ngot %s", "bagjacgzak6wdjvshdtb7lrvlteweyd7f5qjr3dmzmh7g2xpi4xrwoujsio2a", ethTxTrie.elements[1].(cid.Cid).String())
}
}
func TestTxTrieDecodeLeaf(t *testing.T) {
ethTxTrie := prepareDecodedEthTxTrieLeaf(t)
if ethTxTrie.nodeKind != "leaf" {
t.Fatalf("Wrong nodeKind\r\nexpected %s\r\ngot %s", "leaf", ethTxTrie.nodeKind)
}
if len(ethTxTrie.elements) != 2 {
t.Fatalf("Wrong number of elements for a leaf node\r\nexpected %d\r\ngot %d", 2, len(ethTxTrie.elements))
}
if fmt.Sprintf("%x", ethTxTrie.elements[0].([]byte)) != "" {
t.Fatalf("Wrong key\r\nexpected %s\r\ngot %s", "", fmt.Sprintf("%x", ethTxTrie.elements[0].([]byte)))
}
if _, ok := ethTxTrie.elements[1].(*EthTx); !ok {
t.Fatal("Expected element to be an EthTx")
}
if ethTxTrie.elements[1].(*EthTx).String() !=
"<EthereumTx bagjqcgzaqsbvff5xrqh5lobxmhuharvkqdc4jmsqfalsu2xs4pbyix7dvfzq>" {
t.Fatalf("Wrong String()\r\nexpected %s\r\ngot %s", "<EthereumTx bagjqcgzaqsbvff5xrqh5lobxmhuharvkqdc4jmsqfalsu2xs4pbyix7dvfzq>", ethTxTrie.elements[1].(*EthTx).String())
}
}
func TestTxTrieDecodeBranch(t *testing.T) {
ethTxTrie := prepareDecodedEthTxTrieBranch(t)
if ethTxTrie.nodeKind != "branch" {
t.Fatalf("Wrong nodeKind\r\nexpected %s\r\ngot %s", "branch", ethTxTrie.nodeKind)
}
if len(ethTxTrie.elements) != 17 {
t.Fatalf("Wrong number of elements for a branch node\r\nexpected %d\r\ngot %d", 17, len(ethTxTrie.elements))
}
for i, element := range ethTxTrie.elements {
switch {
case i < 9:
if _, ok := element.(cid.Cid); !ok {
t.Fatal("Expected element to be a cid")
}
continue
default:
if element != nil {
t.Fatal("Expected element to be a nil")
}
}
}
}
/*
Block INTERFACE
*/
func TestEthTxTrieBlockElements(t *testing.T) {
ethTxTrie := prepareDecodedEthTxTrieExtension(t)
if fmt.Sprintf("%x", ethTxTrie.RawData())[:10] != "e4820001a0" {
t.Fatalf("Wrong Data\r\nexpected %s\r\ngot %s", "e4820001a0", fmt.Sprintf("%x", ethTxTrie.RawData())[:10])
}
if ethTxTrie.Cid().String() !=
"bagjacgzaw6ccgrfc3qnrl6joodbjjiet4haufnt2xww725luwgfhijnmg36q" {
t.Fatalf("Wrong Cid\r\nexpected %s\r\ngot %s", "bagjacgzaw6ccgrfc3qnrl6joodbjjiet4haufnt2xww725luwgfhijnmg36q", ethTxTrie.Cid().String())
}
}
func TestEthTxTrieString(t *testing.T) {
ethTxTrie := prepareDecodedEthTxTrieExtension(t)
if ethTxTrie.String() != "<EthereumTxTrie bagjacgzaw6ccgrfc3qnrl6joodbjjiet4haufnt2xww725luwgfhijnmg36q>" {
t.Fatalf("Wrong String()\r\nexpected %s\r\ngot %s", "<EthereumTxTrie bagjacgzaw6ccgrfc3qnrl6joodbjjiet4haufnt2xww725luwgfhijnmg36q>", ethTxTrie.String())
}
}
func TestEthTxTrieLoggable(t *testing.T) {
ethTxTrie := prepareDecodedEthTxTrieExtension(t)
l := ethTxTrie.Loggable()
if _, ok := l["type"]; !ok {
t.Fatal("Loggable map expected the field 'type'")
}
if l["type"] != "eth-tx-trie" {
t.Fatalf("Wrong Loggable 'type' value\r\nexpected %s\r\ngot %s", "eth-tx-trie", l["type"])
}
}
/*
Node INTERFACE
*/
func TestTxTrieResolveExtension(t *testing.T) {
ethTxTrie := prepareDecodedEthTxTrieExtension(t)
_ = ethTxTrie
}
func TestTxTrieResolveLeaf(t *testing.T) {
ethTxTrie := prepareDecodedEthTxTrieLeaf(t)
_ = ethTxTrie
}
func TestTxTrieResolveBranch(t *testing.T) {
ethTxTrie := prepareDecodedEthTxTrieBranch(t)
indexes := []string{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f"}
for j, index := range indexes {
obj, rest, err := ethTxTrie.Resolve([]string{index, "nonce"})
switch {
case j < 9:
_, ok := obj.(*node.Link)
if !ok {
t.Fatalf("Returned object is not a link (index: %d)", j)
}
if rest[0] != "nonce" {
t.Fatalf("Wrong rest of the path returned\r\nexpected %s\r\ngot %s", "nonce", rest[0])
}
if err != nil {
t.Fatal("Error should be nil")
}
default:
if obj != nil {
t.Fatalf("Returned object should have been nil")
}
if rest != nil {
t.Fatalf("Rest of the path returned should be nil")
}
if err.Error() != "no such link in this branch" {
t.Fatalf("Wrong error")
}
}
}
otherSuccessCases := [][]string{
{"0", "1", "banana"},
{"1", "banana"},
{"7bc", "def"},
{"bc", "def"},
}
for i := 0; i < len(otherSuccessCases); i = i + 2 {
osc := otherSuccessCases[i]
expectedRest := otherSuccessCases[i+1]
obj, rest, err := ethTxTrie.Resolve(osc)
_, ok := obj.(*node.Link)
if !ok {
t.Fatalf("Returned object is not a link")
}
for j := range expectedRest {
if rest[j] != expectedRest[j] {
t.Fatalf("Wrong rest of the path returned\r\nexpected %s\r\ngot %s", expectedRest[j], rest[j])
}
}
if err != nil {
t.Fatal("Error should be nil")
}
}
}
func TestTraverseTxTrieWithResolve(t *testing.T) {
var err error
txMap := prepareTxTrieMap(t)
// This is the cid of the tx root at the block 4,139,497
currentNode := txMap["bagjacgzaqolvvlyflkdiylijcu4ts6myxczkb2y3ewxmln5oyrsrkfc4v7ua"]
// This is the path we want to traverse
// the transaction id 256, which is RLP encoded to 820100
var traversePath []string
for _, s := range "820100" {
traversePath = append(traversePath, string(s))
}
traversePath = append(traversePath, "value")
var obj interface{}
for {
obj, traversePath, err = currentNode.Resolve(traversePath)
link, ok := obj.(*node.Link)
if !ok {
break
}
if err != nil {
t.Fatal("Error should be nil")
}
currentNode = txMap[link.Cid.String()]
if currentNode == nil {
t.Fatal("transaction trie node not found in memory map")
}
}
if fmt.Sprintf("%v", obj) != "0xc495a958603400" {
t.Fatalf("Wrong value\r\nexpected %s\r\ngot %s", "0xc495a958603400", fmt.Sprintf("%v", obj))
}
}
func TestTxTrieTreeBadParams(t *testing.T) {
ethTxTrie := prepareDecodedEthTxTrieBranch(t)
tree := ethTxTrie.Tree("non-empty-string", 0)
if tree != nil {
t.Fatal("Expected nil to be returned")
}
tree = ethTxTrie.Tree("non-empty-string", 1)
if tree != nil {
t.Fatal("Expected nil to be returned")
}
tree = ethTxTrie.Tree("", 0)
if tree != nil {
t.Fatal("Expected nil to be returned")
}
}
func TestTxTrieTreeExtension(t *testing.T) {
ethTxTrie := prepareDecodedEthTxTrieExtension(t)
tree := ethTxTrie.Tree("", -1)
if len(tree) != 1 {
t.Fatalf("An extension should have one element")
}
if tree[0] != "01" {
t.Fatalf("Wrong trie element\r\nexpected %s\r\ngot %s", "01", tree[0])
}
}
func TestTxTrieTreeBranch(t *testing.T) {
ethTxTrie := prepareDecodedEthTxTrieBranch(t)
tree := ethTxTrie.Tree("", -1)
lookupElements := map[string]interface{}{
"0": nil,
"1": nil,
"2": nil,
"3": nil,
"4": nil,
"5": nil,
"6": nil,
"7": nil,
"8": nil,
}
if len(tree) != len(lookupElements) {
t.Fatalf("Wrong number of elements\r\nexpected %d\r\ngot %d", len(lookupElements), len(tree))
}
for _, te := range tree {
if _, ok := lookupElements[te]; !ok {
t.Fatalf("Unexpected Element: %v", te)
}
}
}
func TestTxTrieLinksBranch(t *testing.T) {
ethTxTrie := prepareDecodedEthTxTrieBranch(t)
desiredValues := []string{
"bagjacgzakhtcfpja453ydiaqxgidqmxhh7jwmxujib663deebwfs3m2n3hoa",
"bagjacgza2p2fuqh4vumknq6x5w7i47usvtu5ixqins6qjjtcks4zge3vx3qq",
"bagjacgza4fkhn7et3ra66yjkzbtvbxjefuketda6jctlut6it7gfahxhywga",
"bagjacgzacnryeybs52xryrka5uxi4eg4hi2mh66esaghu7cetzu6fsukrynq",
"bagjacgzastu5tc7lwz4ap3gznjwkyyepswquub7gvhags5mgdyfynnwbi43a",
"bagjacgza5qgp76ovvorkydni2lchew6ieu5wb55w6hdliiu6vft7zlxtdhjq",
"bagjacgzafnssc4yvln6zxmks5roskw4ckngta5n4yfy2skhlu435ve4b575a",
"bagjacgzagkuei7qxfxefufme2d3xizxokkq4ad3rzl2x4dq2uao6dcr4va2a",
"bagjacgzaxpaehtananrdxjghwukh2wwkkzcqwveppf6xclkrtd26rm27kqwq",
}
links := ethTxTrie.Links()
for i, v := range desiredValues {
if links[i].Cid.String() != v {
t.Fatalf("Wrong cid for link %d\r\nexpected %s\r\ngot %s", i, v, links[i].Cid.String())
}
}
}
/*
EthTxTrie Functions
*/
func TestTxTrieJSONMarshalExtension(t *testing.T) {
ethTxTrie := prepareDecodedEthTxTrieExtension(t)
jsonOutput, err := ethTxTrie.MarshalJSON()
checkError(err, t)
var data map[string]interface{}
err = json.Unmarshal(jsonOutput, &data)
checkError(err, t)
if parseMapElement(data["01"]) !=
"bagjacgzak6wdjvshdtb7lrvlteweyd7f5qjr3dmzmh7g2xpi4xrwoujsio2a" {
t.Fatalf("Wrong Marshaled Value\r\nexpected %s\r\ngot %s", "bagjacgzak6wdjvshdtb7lrvlteweyd7f5qjr3dmzmh7g2xpi4xrwoujsio2a", parseMapElement(data["01"]))
}
if data["type"] != "extension" {
t.Fatalf("Wrong node type\r\nexpected %s\r\ngot %s", "extension", data["type"])
}
}
func TestTxTrieJSONMarshalLeaf(t *testing.T) {
ethTxTrie := prepareDecodedEthTxTrieLeaf(t)
jsonOutput, err := ethTxTrie.MarshalJSON()
checkError(err, t)
var data map[string]interface{}
err = json.Unmarshal(jsonOutput, &data)
checkError(err, t)
if data["type"] != "leaf" {
t.Fatalf("Wrong node type\r\nexpected %s\r\ngot %s", "leaf", data["type"])
}
if fmt.Sprintf("%v", data[""].(map[string]interface{})["nonce"]) !=
"40243" {
t.Fatalf("Wrong nonce value\r\nexepcted %s\r\ngot %s", "40243", fmt.Sprintf("%v", data[""].(map[string]interface{})["nonce"]))
}
}
func TestTxTrieJSONMarshalBranch(t *testing.T) {
ethTxTrie := prepareDecodedEthTxTrieBranch(t)
jsonOutput, err := ethTxTrie.MarshalJSON()
checkError(err, t)
var data map[string]interface{}
err = json.Unmarshal(jsonOutput, &data)
checkError(err, t)
desiredValues := map[string]string{
"0": "bagjacgzakhtcfpja453ydiaqxgidqmxhh7jwmxujib663deebwfs3m2n3hoa",
"1": "bagjacgza2p2fuqh4vumknq6x5w7i47usvtu5ixqins6qjjtcks4zge3vx3qq",
"2": "bagjacgza4fkhn7et3ra66yjkzbtvbxjefuketda6jctlut6it7gfahxhywga",
"3": "bagjacgzacnryeybs52xryrka5uxi4eg4hi2mh66esaghu7cetzu6fsukrynq",
"4": "bagjacgzastu5tc7lwz4ap3gznjwkyyepswquub7gvhags5mgdyfynnwbi43a",
"5": "bagjacgza5qgp76ovvorkydni2lchew6ieu5wb55w6hdliiu6vft7zlxtdhjq",
"6": "bagjacgzafnssc4yvln6zxmks5roskw4ckngta5n4yfy2skhlu435ve4b575a",
"7": "bagjacgzagkuei7qxfxefufme2d3xizxokkq4ad3rzl2x4dq2uao6dcr4va2a",
"8": "bagjacgzaxpaehtananrdxjghwukh2wwkkzcqwveppf6xclkrtd26rm27kqwq",
}
for k, v := range desiredValues {
if parseMapElement(data[k]) != v {
t.Fatalf("Wrong Marshaled Value %s\r\nexpected %s\r\ngot %s", k, v, parseMapElement(data[k]))
}
}
for _, v := range []string{"a", "b", "c", "d", "e", "f"} {
if data[v] != nil {
t.Fatal("Expected value to be nil")
}
}
if data["type"] != "branch" {
t.Fatalf("Wrong node type\r\nexpected %s\r\ngot %s", "branch", data["type"])
}
}
/*
AUXILIARS
*/
// prepareDecodedEthTxTrie simulates an IPLD block available in the datastore,
// checks the source RLP and tests for the absence of errors during the decoding fase.
func prepareDecodedEthTxTrie(branchDataRLP string, t *testing.T) *EthTxTrie {
b, err := hex.DecodeString(branchDataRLP)
checkError(err, t)
c, err := RawdataToCid(MEthTxTrie, b, multihash.KECCAK_256)
checkError(err, t)
storedEthTxTrie, err := block.NewBlockWithCid(b, c)
checkError(err, t)
ethTxTrie, err := DecodeEthTxTrie(storedEthTxTrie.Cid(), storedEthTxTrie.RawData())
checkError(err, t)
return ethTxTrie
}
func prepareDecodedEthTxTrieExtension(t *testing.T) *EthTxTrie {
extensionDataRLP :=
"e4820001a057ac34d6471cc3f5c6ab992c4c0fe5ec131d8d9961fe6d5de8e5e367513243b4"
return prepareDecodedEthTxTrie(extensionDataRLP, t)
}
func prepareDecodedEthTxTrieLeaf(t *testing.T) *EthTxTrie {
leafDataRLP :=
"f87220b86ff86d829d3384ee6b280083015f9094e0e6c781b8cba08bc840" +
"7eac0101b668d1fa6f4987c495a9586034008026a0981b6223c9d3c31971" +
"6da3cf057da84acf0fef897f4003d8a362d7bda42247dba066be134c4bc4" +
"32125209b5056ef274b7423bcac7cc398cf60b83aaff7b95469f"
return prepareDecodedEthTxTrie(leafDataRLP, t)
}
func prepareDecodedEthTxTrieBranch(t *testing.T) *EthTxTrie {
branchDataRLP :=
"f90131a051e622bd20e77781a010b9903832e73fd3665e89407ded8c840d8b2db34dd9" +
"dca0d3f45a40fcad18a6c3d7edbe8e7e92ace9d45e086cbd04a66254b9931375bee1a0" +
"e15476fc93dc41ef612ac86750dd242d14498c1e48a6ba4fc89fcc501ee7c58ca01363" +
"826032eeaf1c4540ed2e8e10dc3a34c3fbc4900c7a7c449e69e2ca8a8e1ba094e9d98b" +
"ebb67807ecd96a6cac608f95a14a07e6a9c06975861e0b86b6c14736a0ec0cfff9d5ab" +
"a2ac0da8d2c4725bc8253b60f7b6f1c6b4229ea967fcaef319d3a02b652173155b7d9b" +
"b152ec5d255b82534d3075bcc171a928eba737da9381effaa032a8447e172dc85a1584" +
"d0f77466ee52a1c00f71caf57e0e1aa01de18a3ca834a0bbc043cc0d03623ba4c7b514" +
"7d5aca56450b548f797d712d5198f5e8b35f542d8080808080808080"
return prepareDecodedEthTxTrie(branchDataRLP, t)
}
func prepareTxTrieMap(t *testing.T) map[string]*EthTxTrie {
fi, err := os.Open("test_data/eth-block-body-json-4139497")
checkError(err, t)
_, _, txTrieNodes, err := FromBlockJSON(fi)
checkError(err, t)
out := make(map[string]*EthTxTrie)
for _, txTrieNode := range txTrieNodes {
decodedNode, err := DecodeEthTxTrie(txTrieNode.Cid(), txTrieNode.RawData())
checkError(err, t)
out[txTrieNode.Cid().String()] = decodedNode
}
return out
}

211
statediff/indexer/ipfs/ipld/shared.go Normal file
View File

@ -0,0 +1,211 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipld
import (
"bytes"
"errors"
"github.com/ipfs/go-cid"
mh "github.com/multiformats/go-multihash"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/rlp"
sdtrie "github.com/ethereum/go-ethereum/statediff/trie"
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
"github.com/ethereum/go-ethereum/trie"
)
// IPLD Codecs for Ethereum
// See the authoritative document:
// https://github.com/multiformats/multicodec/blob/master/table.csv
const (
RawBinary = 0x55
MEthHeader = 0x90
MEthHeaderList = 0x91
MEthTxTrie = 0x92
MEthTx = 0x93
MEthTxReceiptTrie = 0x94
MEthTxReceipt = 0x95
MEthStateTrie = 0x96
MEthAccountSnapshot = 0x97
MEthStorageTrie = 0x98
MEthLogTrie = 0x99
MEthLog = 0x9a
)
var (
nullHashBytes = common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000000")
ErrInvalidLink = errors.New("no such link")
)
// RawdataToCid takes the desired codec and a slice of bytes
// and returns the proper cid of the object.
func RawdataToCid(codec uint64, rawdata []byte, multiHash uint64) (cid.Cid, error) {
c, err := cid.Prefix{
Codec: codec,
Version: 1,
MhType: multiHash,
MhLength: -1,
}.Sum(rawdata)
if err != nil {
return cid.Cid{}, err
}
return c, nil
}
// keccak256ToCid takes a keccak256 hash and returns its cid based on
// the codec given.
func keccak256ToCid(codec uint64, h []byte) cid.Cid {
buf, err := mh.Encode(h, mh.KECCAK_256)
if err != nil {
panic(err)
}
return cid.NewCidV1(codec, mh.Multihash(buf))
}
// commonHashToCid takes a go-ethereum common.Hash and returns its
// cid based on the codec given,
func commonHashToCid(codec uint64, h common.Hash) cid.Cid {
mhash, err := mh.Encode(h[:], mh.KECCAK_256)
if err != nil {
panic(err)
}
return cid.NewCidV1(codec, mhash)
}
// localTrie wraps a go-ethereum trie and its underlying memory db.
// It contributes to the creation of the trie node objects.
type localTrie struct {
db ethdb.Database
trieDB *trie.Database
trie *trie.Trie
}
// newLocalTrie initializes and returns a localTrie object
func newLocalTrie() *localTrie {
var err error
lt := &localTrie{}
lt.db = rawdb.NewMemoryDatabase()
lt.trieDB = trie.NewDatabase(lt.db)
lt.trie, err = trie.New(common.Hash{}, lt.trieDB)
if err != nil {
panic(err)
}
return lt
}
// Add receives the index of an object and its rawdata value
// and includes it into the localTrie
func (lt *localTrie) Add(idx int, rawdata []byte) error {
key, err := rlp.EncodeToBytes(uint(idx))
if err != nil {
panic(err)
}
return lt.trie.TryUpdate(key, rawdata)
}
// rootHash returns the computed trie root.
// Useful for sanity checks on parsed data.
func (lt *localTrie) rootHash() []byte {
return lt.trie.Hash().Bytes()
}
func (lt *localTrie) commit() error {
// commit trie nodes to trieDB
var err error
_, _, err = lt.trie.Commit(nil)
if err != nil {
return err
}
// commit trieDB to the underlying ethdb.Database
if err := lt.trieDB.Commit(lt.trie.Hash(), false, nil); err != nil {
return err
}
return nil
}
// getKeys returns the stored keys of the memory database
// of the localTrie for further processing.
func (lt *localTrie) getKeys() ([][]byte, error) {
if err := lt.commit(); err != nil {
return nil, err
}
// collect all of the node keys
it := lt.trie.NodeIterator([]byte{})
keyBytes := make([][]byte, 0)
for it.Next(true) {
if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
continue
}
keyBytes = append(keyBytes, it.Hash().Bytes())
}
return keyBytes, nil
}
type nodeKey struct {
dbKey []byte
TrieKey []byte
}
// getLeafKeys returns the stored leaf keys from the memory database
// of the localTrie for further processing.
func (lt *localTrie) getLeafKeys() ([]*nodeKey, error) {
if err := lt.commit(); err != nil {
return nil, err
}
it := lt.trie.NodeIterator([]byte{})
leafKeys := make([]*nodeKey, 0)
for it.Next(true) {
if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
continue
}
node, nodeElements, err := sdtrie.ResolveNode(it, lt.trieDB)
if err != nil {
return nil, err
}
if node.NodeType != sdtypes.Leaf {
continue
}
partialPath := trie.CompactToHex(nodeElements[0].([]byte))
valueNodePath := append(node.Path, partialPath...)
encodedPath := trie.HexToCompact(valueNodePath)
leafKey := encodedPath[1:]
leafKeys = append(leafKeys, &nodeKey{dbKey: it.Hash().Bytes(), TrieKey: leafKey})
}
return leafKeys, nil
}
// getRLP encodes the given object to RLP returning its bytes.
func getRLP(object interface{}) []byte {
buf := new(bytes.Buffer)
if err := rlp.Encode(buf, object); err != nil {
panic(err)
}
return buf.Bytes()
}

1
statediff/indexer/ipfs/ipld/test_data/error-tx-eth-block-body-json-999999 Normal file
View File

File diff suppressed because one or more lines are too long

1
statediff/indexer/ipfs/ipld/test_data/eth-block-body-json-0 Normal file
View File

@ -0,0 +1 @@
{"jsonrpc":"2.0","id":1,"result":{"author":"0x0000000000000000000000000000000000000000","difficulty":"0x400000000","extraData":"0x11bbe8db4e347b4e8c937c1c8370e4b5ed33adb3db69cbdb7a38e1e50b1b82fa","gasLimit":"0x1388","gasUsed":"0x0","hash":"0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3","logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","miner":"0x0000000000000000000000000000000000000000","mixHash":"0x0000000000000000000000000000000000000000000000000000000000000000","nonce":"0x0000000000000042","number":"0x0","parentHash":"0x0000000000000000000000000000000000000000000000000000000000000000","receiptsRoot":"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421","sealFields":["0x0000000000000000000000000000000000000000000000000000000000000000","0x0000000000000042"],"sha3Uncles":"0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347","size":"0x21c","stateRoot":"0xd7f8974fb5ac78d9ac099b9ad5018bedc2ce0a72dad1827a1709da30580f0544","timestamp":"0x0","totalDifficulty":"0x400000000","transactions":[],"transactionsRoot":"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421","uncles":[]}}

1
statediff/indexer/ipfs/ipld/test_data/eth-block-body-json-4139497 Normal file
View File

File diff suppressed because one or more lines are too long

1
statediff/indexer/ipfs/ipld/test_data/eth-block-body-json-997522 Normal file
View File

@ -0,0 +1 @@
{"jsonrpc":"2.0","result":{"author":"0x4bb96091ee9d802ed039c4d1a5f6216f90f81b01","difficulty":"0xae22b2113ed","extraData":"0xd783010400844765746887676f312e352e31856c696e7578","gasLimit":"0x2fefd8","gasUsed":"0x5208","hash":"0x79851e1adb52a8c5490da2df5d8c060b1cc44a3b6eeaada2e20edba5a8e84523","logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","miner":"0x4bb96091ee9d802ed039c4d1a5f6216f90f81b01","mixHash":"0x2565992ba4dbd7ab3bb08d1da34051ae1d90c79bc637a21aa2f51f6380bf5f6a","nonce":"0xf7a14147c2320b2d","number":"0xf3892","parentHash":"0x8ad6d5cbe7ec75ed71d5153dd58f2fd413b17c398ad2a7d9309459ce884e6c9b","receiptsRoot":"0xa73a95d90de29c66220c8b8da825cf34ae969efc7f9a878d8ed893565e4b4676","sealFields":["0xa02565992ba4dbd7ab3bb08d1da34051ae1d90c79bc637a21aa2f51f6380bf5f6a","0x88f7a14147c2320b2d"],"sha3Uncles":"0x08793b633d0b21b980107f3e3277c6693f2f3739e0c676a238cbe24d9ae6e252","size":"0x6c0","stateRoot":"0x11e5ea49ecbee25a9b8f267492a5d296ac09cf6179b43bc334242d052bac5963","timestamp":"0x56bf10c5","totalDifficulty":"0x629a0a89232bcd5b","transactions":[{"blockHash":"0x79851e1adb52a8c5490da2df5d8c060b1cc44a3b6eeaada2e20edba5a8e84523","blockNumber":"0xf3892","condition":null,"creates":null,"from":"0x4bb96091ee9d802ed039c4d1a5f6216f90f81b01","gas":"0x15f90","gasPrice":"0xa","hash":"0xd0fc6b051f16468862c462c672532427efef537ea3737b25b10716949d0e2228","input":"0x","networkId":null,"nonce":"0x7c37","publicKey":"0xa9177f27b99a4ad938359d77e0dca4b64e7ce3722c835d8087d4eecb27c8a54d59e2917e6b31ec12e44b1064d102d35815f9707af9571f15e92d1b6fbcd207e9","r":"0x76933e91718154f18db2e993bc96e82abd9a0fac2bae284875341cbecafa837b","raw":"0xf86a827c370a83015f909404a6c6a293340fc3f2244d097b0cfd84d5317ba58844b1eec616322c1c801ba076933e91718154f18db2e993bc96e82abd9a0fac2bae284875341cbecafa837ba02f165c2c4b5f4b786a95e106c48bccc7e065647af5a1942025b6fbfafeabbbf6","s":"0x2f165c2c4b5f4b786a95e106c48bccc7e065647af5a1942025b6fbfafeabbbf6","standardV":"0x0","to":"0x04a6c6a293340fc3f2244d097b0cfd84d5317ba5","transactionIndex":"0x0","v":"0x1b","value":"0x44b1eec616322c1c"}],"transactionsRoot":"0x7ab22cfcf6db5d1628ac888c25e6bc49aba2faaa200fc880f800f1db1e8bd3cc","uncles":["0x319e0dc9a53711579c4ba88062c927a0045443cca57625903ef471d760506a94","0x0324272e484e509c3c9e9e75ad8b48c7d34556e6b269dd72331033fd5cdc1b2a"]},"id":1}

1
statediff/indexer/ipfs/ipld/test_data/eth-block-body-json-999998 Normal file
View File

File diff suppressed because one or more lines are too long

1
statediff/indexer/ipfs/ipld/test_data/eth-block-body-json-999999 Normal file
View File

File diff suppressed because one or more lines are too long

BIN
statediff/indexer/ipfs/ipld/test_data/eth-block-body-rlp-997522 Normal file
View File

Binary file not shown.

BIN
statediff/indexer/ipfs/ipld/test_data/eth-block-body-rlp-999999 Normal file
View File

Binary file not shown.

BIN
statediff/indexer/ipfs/ipld/test_data/eth-block-header-rlp-999996 Normal file
View File

Binary file not shown.

BIN
statediff/indexer/ipfs/ipld/test_data/eth-block-header-rlp-999997 Normal file
View File

Binary file not shown.

BIN
statediff/indexer/ipfs/ipld/test_data/eth-block-header-rlp-999999 Normal file
View File

Binary file not shown.

BIN
statediff/indexer/ipfs/ipld/test_data/eth-state-trie-rlp-0e8b34 Normal file
View File

Binary file not shown.

1
statediff/indexer/ipfs/ipld/test_data/eth-state-trie-rlp-56864f Normal file
View File

@ -0,0 +1 @@
â FKˇd?¶fç_‹¦·YA( "a<13>î2–cUSyI

5
statediff/indexer/ipfs/ipld/test_data/eth-state-trie-rlp-6fc2d7 Normal file
View File

@ -0,0 +1,5 @@
ù Úä<C39A>[G“ (»o½Uå,ÔrBÇõSsµ^²€^âä©) 7ó7ì€.Ytâç5_ñ¤ƒ+9¸FÙÃYzFv Ú<C2A0>b{¸ûî­³à¨äõ(Û1Y¶«-í¤©÷Ê<C3B7>*µ —f&HÕ‚•ÐЪK€UX<55> v• R€%IÙJ/ ÌÇïä³A?Ö¦lŸ@éU¯wFI¨Ùý!-jZ9Ý»g Öͳ.+Ö5î/ž¼”ݽ ±À<>fbŽfzìW  [‰ =É@æúpì­NÐIÓ¥º ¨ùÀRRVíI Ÿ ¸B'Ô<>öŠìÇr“šY¯©á¤«W<C2AB>{i‹Û‰â›`DfŽ ý™ p¹JÎW䌿e¡j§pÆEùõﺇ»å<C2BB>
) áj|ΦtŠé é/Šï;=ÂH¥W¹¬N)i41?$÷üí_ B7<ô 0ÙMé
#¸óŒík|¸¸’_î<5F> *(¢Z _‰ÒAÿBˆd÷ˆ˜fHLïb-å:Fç•ßÞÃ61Ÿ u— fE&ÈǕ΢{‹rE\IeqàEURÛÀhź1 Õ¾<C395>‰/Ú,XZ–˜Ž¥ïÍ:˜Ž
†‚ iK7Å ÷°5.8ò١MQºêMÞáw tÈâ 5R3ÃÈ<C383>În I¿n<C2BF>ð¬¯Ðïømïî³VŽDÕ-"5Ï4
á\`4â²A€

BIN
statediff/indexer/ipfs/ipld/test_data/eth-state-trie-rlp-727994 Normal file
View File

Binary file not shown.

BIN
statediff/indexer/ipfs/ipld/test_data/eth-state-trie-rlp-c9070d Normal file
View File

Binary file not shown.

BIN
statediff/indexer/ipfs/ipld/test_data/eth-state-trie-rlp-d5be90 Normal file
View File

Binary file not shown.

BIN
statediff/indexer/ipfs/ipld/test_data/eth-state-trie-rlp-d7f897 Normal file
View File

Binary file not shown.

BIN
statediff/indexer/ipfs/ipld/test_data/eth-state-trie-rlp-eb2f5f Normal file
View File

Binary file not shown.

BIN
statediff/indexer/ipfs/ipld/test_data/eth-storage-trie-rlp-000dd0 Normal file
View File

Binary file not shown.

1
statediff/indexer/ipfs/ipld/test_data/eth-storage-trie-rlp-113049 Normal file
View File

@ -0,0 +1 @@
β ¤ξJN…>λb$Ιkg<6B>Ί­$α2ζΝ |Δι <0A>κdΉ¥

1
statediff/indexer/ipfs/ipld/test_data/eth-storage-trie-rlp-9d1860 Normal file
View File

@ -0,0 +1 @@
<EFBFBD>

BIN
statediff/indexer/ipfs/ipld/test_data/eth-storage-trie-rlp-ffbcad Normal file
View File

Binary file not shown.

1
statediff/indexer/ipfs/ipld/test_data/eth-storage-trie-rlp-ffc25c Normal file
View File

@ -0,0 +1 @@
<EFBFBD>Q<EFBFBD><EFBFBD><EFBFBD><EFBFBD> .ّ<>ٍس<D98D>b<EFBFBD>R<EFBFBD>ّ<EFBFBD><18>f<><66>-<2D>oّt6<74>فKي<4B><D98A><EFBFBD><EFBFBD><EFBFBD> <EFBFBD>ا؟<D8A7>U<EFBFBD><<3C><>مZh<5A>ة <09>hmx[<5B>-­#k<>3حع<D8AD><D8B9><EFBFBD><EFBFBD><EFBFBD><EFBFBD>

1
statediff/indexer/ipfs/ipld/test_data/eth-uncle-json-997522-0 Normal file
View File

@ -0,0 +1 @@
{"jsonrpc":"2.0","result":{"author":"0x68795c4aa09d6f4ed3e5deddf8c2ad3049a601da","difficulty":"0xae387bd92cc","extraData":"0xd783010400844765746887676f312e352e31856c696e7578","gasLimit":"0x2fefd8","gasUsed":"0x0","hash":"0x319e0dc9a53711579c4ba88062c927a0045443cca57625903ef471d760506a94","logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","miner":"0x68795c4aa09d6f4ed3e5deddf8c2ad3049a601da","mixHash":"0x2d4fd3be43fd50f5c0ba7f1c86c8f468b5c14f75b6143da927a2994383f26640","nonce":"0x0aaaa7fe9d7cf7f4","number":"0xf388f","parentHash":"0xac74216bbdb0ebec6612ad5f26301ab50e588aabe75a804bc2068f83980eefc6","receiptsRoot":"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421","sealFields":["0xa02d4fd3be43fd50f5c0ba7f1c86c8f468b5c14f75b6143da927a2994383f26640","0x880aaaa7fe9d7cf7f4"],"sha3Uncles":"0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347","size":null,"stateRoot":"0xf9309492322aab44243f8c38240874b37dd0c563bac85f1a816941acc945b21d","timestamp":"0x56bf1097","totalDifficulty":"0x6299e9e3fdb6eb4d","transactions":[],"transactionsRoot":"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421","uncles":[]},"id":1}

1
statediff/indexer/ipfs/ipld/test_data/eth-uncle-json-997522-1 Normal file
View File

@ -0,0 +1 @@
{"jsonrpc":"2.0","result":{"author":"0x68795c4aa09d6f4ed3e5deddf8c2ad3049a601da","difficulty":"0xae22b4c9b9a","extraData":"0xd783010400844765746887676f312e352e31856c696e7578","gasLimit":"0x2fefd8","gasUsed":"0xf618","hash":"0x0324272e484e509c3c9e9e75ad8b48c7d34556e6b269dd72331033fd5cdc1b2a","logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","miner":"0x68795c4aa09d6f4ed3e5deddf8c2ad3049a601da","mixHash":"0x0f3bdea5170d6af74b70fcf0df81969f6bb1b740f4a6c78df1d354f172865594","nonce":"0x4c691de262b2b3d9","number":"0xf3890","parentHash":"0xcb9efe9bc3c59be7fb673576d661aff9ca75b1522f58fd38d03d3d49b32bddb3","receiptsRoot":"0x5cf73738487f67f1c0a1c2d1083ae014f38e1aab5eb26a8929a511c48b07ea03","sealFields":["0xa00f3bdea5170d6af74b70fcf0df81969f6bb1b740f4a6c78df1d354f172865594","0x884c691de262b2b3d9"],"sha3Uncles":"0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347","size":null,"stateRoot":"0x968e8d8d099572ac783f4511724ec646f59bb33f7395edf858f98b37c8c3b265","timestamp":"0x56bf10b1","totalDifficulty":"0x6299f4c6290386e7","transactions":[],"transactionsRoot":"0x9cea6a59a5df69111ead7406a431c764b2357120e5b61425388df62f87cbcbc3","uncles":[]},"id":1}

BIN
statediff/indexer/ipfs/ipld/test_data/tx_data Normal file
View File

Binary file not shown.

456
statediff/indexer/ipfs/ipld/trie_node.go Normal file
View File

@ -0,0 +1,456 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipld
import (
"encoding/json"
"fmt"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ipfs/go-cid"
node "github.com/ipfs/go-ipld-format"
)
const (
extension = "extension"
leaf = "leaf"
branch = "branch"
)
// TrieNode is the general abstraction for
//ethereum IPLD trie nodes.
type TrieNode struct {
// leaf, extension or branch
nodeKind string
// If leaf or extension: [0] is key, [1] is val.
// If branch: [0] - [16] are children.
elements []interface{}
// IPLD block information
cid cid.Cid
rawdata []byte
}
/*
OUTPUT
*/
type trieNodeLeafDecoder func([]interface{}) ([]interface{}, error)
// decodeTrieNode returns a TrieNode object from an IPLD block's
// cid and rawdata.
func decodeTrieNode(c cid.Cid, b []byte,
leafDecoder trieNodeLeafDecoder) (*TrieNode, error) {
var (
i, decoded, elements []interface{}
nodeKind string
err error
)
if err = rlp.DecodeBytes(b, &i); err != nil {
return nil, err
}
codec := c.Type()
switch len(i) {
case 2:
nodeKind, decoded, err = decodeCompactKey(i)
if err != nil {
return nil, err
}
if nodeKind == extension {
elements, err = parseTrieNodeExtension(decoded, codec)
if err != nil {
return nil, err
}
}
if nodeKind == leaf {
elements, err = leafDecoder(decoded)
if err != nil {
return nil, err
}
}
if nodeKind != extension && nodeKind != leaf {
return nil, fmt.Errorf("unexpected nodeKind returned from decoder")
}
case 17:
nodeKind = branch
elements, err = parseTrieNodeBranch(i, codec)
if err != nil {
return nil, err
}
default:
return nil, fmt.Errorf("unknown trie node type")
}
return &TrieNode{
nodeKind: nodeKind,
elements: elements,
rawdata: b,
cid: c,
}, nil
}
// decodeCompactKey takes a compact key, and returns its nodeKind and value.
func decodeCompactKey(i []interface{}) (string, []interface{}, error) {
first := i[0].([]byte)
last := i[1].([]byte)
switch first[0] / 16 {
case '\x00':
return extension, []interface{}{
nibbleToByte(first)[2:],
last,
}, nil
case '\x01':
return extension, []interface{}{
nibbleToByte(first)[1:],
last,
}, nil
case '\x02':
return leaf, []interface{}{
nibbleToByte(first)[2:],
last,
}, nil
case '\x03':
return leaf, []interface{}{
nibbleToByte(first)[1:],
last,
}, nil
default:
return "", nil, fmt.Errorf("unknown hex prefix")
}
}
// parseTrieNodeExtension helper improves readability
func parseTrieNodeExtension(i []interface{}, codec uint64) ([]interface{}, error) {
return []interface{}{
i[0].([]byte),
keccak256ToCid(codec, i[1].([]byte)),
}, nil
}
// parseTrieNodeBranch helper improves readability
func parseTrieNodeBranch(i []interface{}, codec uint64) ([]interface{}, error) {
var out []interface{}
for i, vi := range i {
v, ok := vi.([]byte)
// Sometimes this throws "panic: interface conversion: interface {} is []interface {}, not []uint8"
// Figure out why, and if it is okay to continue
if !ok {
return nil, fmt.Errorf("unable to decode branch node entry into []byte at position: %d value: %+v", i, vi)
}
switch len(v) {
case 0:
out = append(out, nil)
case 32:
out = append(out, keccak256ToCid(codec, v))
default:
return nil, fmt.Errorf("unrecognized object: %v", v)
}
}
return out, nil
}
/*
Node INTERFACE
*/
// Resolve resolves a path through this node, stopping at any link boundary
// and returning the object found as well as the remaining path to traverse
func (t *TrieNode) Resolve(p []string) (interface{}, []string, error) {
switch t.nodeKind {
case extension:
return t.resolveTrieNodeExtension(p)
case leaf:
return t.resolveTrieNodeLeaf(p)
case branch:
return t.resolveTrieNodeBranch(p)
default:
return nil, nil, fmt.Errorf("nodeKind case not implemented")
}
}
// Tree lists all paths within the object under 'path', and up to the given depth.
// To list the entire object (similar to `find .`) pass "" and -1
func (t *TrieNode) Tree(p string, depth int) []string {
if p != "" || depth == 0 {
return nil
}
var out []string
switch t.nodeKind {
case extension:
var val string
for _, e := range t.elements[0].([]byte) {
val += fmt.Sprintf("%x", e)
}
return []string{val}
case branch:
for i, elem := range t.elements {
if _, ok := elem.(cid.Cid); ok {
out = append(out, fmt.Sprintf("%x", i))
}
}
return out
default:
return nil
}
}
// ResolveLink is a helper function that calls resolve and asserts the
// output is a link
func (t *TrieNode) ResolveLink(p []string) (*node.Link, []string, error) {
obj, rest, err := t.Resolve(p)
if err != nil {
return nil, nil, err
}
lnk, ok := obj.(*node.Link)
if !ok {
return nil, nil, fmt.Errorf("was not a link")
}
return lnk, rest, nil
}
// Copy will go away. It is here to comply with the interface.
func (t *TrieNode) Copy() node.Node {
panic("implement me")
}
// Links is a helper function that returns all links within this object
func (t *TrieNode) Links() []*node.Link {
var out []*node.Link
for _, i := range t.elements {
c, ok := i.(cid.Cid)
if ok {
out = append(out, &node.Link{Cid: c})
}
}
return out
}
// Stat will go away. It is here to comply with the interface.
func (t *TrieNode) Stat() (*node.NodeStat, error) {
return &node.NodeStat{}, nil
}
// Size will go away. It is here to comply with the interface.
func (t *TrieNode) Size() (uint64, error) {
return 0, nil
}
/*
TrieNode functions
*/
// MarshalJSON processes the transaction trie into readable JSON format.
func (t *TrieNode) MarshalJSON() ([]byte, error) {
var out map[string]interface{}
switch t.nodeKind {
case extension:
fallthrough
case leaf:
var hexPrefix string
for _, e := range t.elements[0].([]byte) {
hexPrefix += fmt.Sprintf("%x", e)
}
// if we got a byte we need to do this casting otherwise
// it will be marshaled to a base64 encoded value
if _, ok := t.elements[1].([]byte); ok {
var hexVal string
for _, e := range t.elements[1].([]byte) {
hexVal += fmt.Sprintf("%x", e)
}
t.elements[1] = hexVal
}
out = map[string]interface{}{
"type": t.nodeKind,
hexPrefix: t.elements[1],
}
case branch:
out = map[string]interface{}{
"type": branch,
"0": t.elements[0],
"1": t.elements[1],
"2": t.elements[2],
"3": t.elements[3],
"4": t.elements[4],
"5": t.elements[5],
"6": t.elements[6],
"7": t.elements[7],
"8": t.elements[8],
"9": t.elements[9],
"a": t.elements[10],
"b": t.elements[11],
"c": t.elements[12],
"d": t.elements[13],
"e": t.elements[14],
"f": t.elements[15],
}
default:
return nil, fmt.Errorf("nodeKind %s not supported", t.nodeKind)
}
return json.Marshal(out)
}
// nibbleToByte expands the nibbles of a byte slice into their own bytes.
func nibbleToByte(k []byte) []byte {
var out []byte
for _, b := range k {
out = append(out, b/16)
out = append(out, b%16)
}
return out
}
// Resolve reading conveniences
func (t *TrieNode) resolveTrieNodeExtension(p []string) (interface{}, []string, error) {
nibbles := t.elements[0].([]byte)
idx, rest := shiftFromPath(p, len(nibbles))
if len(idx) < len(nibbles) {
return nil, nil, fmt.Errorf("not enough nibbles to traverse this extension")
}
for _, i := range idx {
if getHexIndex(string(i)) == -1 {
return nil, nil, fmt.Errorf("invalid path element")
}
}
for i, n := range nibbles {
if string(idx[i]) != fmt.Sprintf("%x", n) {
return nil, nil, fmt.Errorf("no such link in this extension")
}
}
return &node.Link{Cid: t.elements[1].(cid.Cid)}, rest, nil
}
func (t *TrieNode) resolveTrieNodeLeaf(p []string) (interface{}, []string, error) {
nibbles := t.elements[0].([]byte)
if len(nibbles) != 0 {
idx, rest := shiftFromPath(p, len(nibbles))
if len(idx) < len(nibbles) {
return nil, nil, fmt.Errorf("not enough nibbles to traverse this leaf")
}
for _, i := range idx {
if getHexIndex(string(i)) == -1 {
return nil, nil, fmt.Errorf("invalid path element")
}
}
for i, n := range nibbles {
if string(idx[i]) != fmt.Sprintf("%x", n) {
return nil, nil, fmt.Errorf("no such link in this extension")
}
}
p = rest
}
link, ok := t.elements[1].(node.Node)
if !ok {
return nil, nil, fmt.Errorf("leaf children is not an IPLD node")
}
return link.Resolve(p)
}
func (t *TrieNode) resolveTrieNodeBranch(p []string) (interface{}, []string, error) {
idx, rest := shiftFromPath(p, 1)
hidx := getHexIndex(idx)
if hidx == -1 {
return nil, nil, fmt.Errorf("incorrect path")
}
child := t.elements[hidx]
if child != nil {
return &node.Link{Cid: child.(cid.Cid)}, rest, nil
}
return nil, nil, fmt.Errorf("no such link in this branch")
}
// shiftFromPath extracts from a given path (as a slice of strings)
// the given number of elements as a single string, returning whatever
// it has not taken.
//
// Examples:
// ["0", "a", "something"] and 1 -> "0" and ["a", "something"]
// ["ab", "c", "d", "1"] and 2 -> "ab" and ["c", "d", "1"]
// ["abc", "d", "1"] and 2 -> "ab" and ["c", "d", "1"]
func shiftFromPath(p []string, i int) (string, []string) {
var (
out string
rest []string
)
for _, pe := range p {
re := ""
for _, c := range pe {
if len(out) < i {
out += string(c)
} else {
re += string(c)
}
}
if len(out) == i && re != "" {
rest = append(rest, re)
}
}
return out, rest
}
// getHexIndex returns to you the integer 0 - 15 equivalent to your
// string character if applicable, or -1 otherwise.
func getHexIndex(s string) int {
if len(s) != 1 {
return -1
}
c := s[0]
switch {
case '0' <= c && c <= '9':
return int(c - '0')
case 'a' <= c && c <= 'f':
return int(c - 'a' + 10)
}
return -1
}

22
statediff/indexer/ipfs/models.go Normal file
View File

@ -0,0 +1,22 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipfs
type BlockModel struct {
CID string `db:"key"`
Data []byte `db:"data"`
}

BIN
statediff/indexer/mainnet_tests/block_12579670.rlp Normal file
View File

Binary file not shown.

BIN
statediff/indexer/mainnet_tests/block_12600011.rlp Normal file
View File

Binary file not shown.

BIN
statediff/indexer/mainnet_tests/block_12619985.rlp Normal file
View File

Binary file not shown.

BIN
statediff/indexer/mainnet_tests/block_12625121.rlp Normal file
View File

Binary file not shown.

BIN
statediff/indexer/mainnet_tests/block_12655432.rlp Normal file
View File

Binary file not shown.

BIN
statediff/indexer/mainnet_tests/block_12914664.rlp Normal file
View File

Binary file not shown.

107
statediff/indexer/mainnet_tests/indexer_test.go Normal file
View File

@ -0,0 +1,107 @@
// VulcanizeDB
// Copyright © 2021 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package mainnet_tests
import (
"fmt"
"math/big"
"os"
"testing"
"github.com/stretchr/testify/require"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/statediff/indexer"
"github.com/ethereum/go-ethereum/statediff/indexer/mocks"
"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
)
var (
err error
db *postgres.DB
chainConf = params.MainnetChainConfig
)
func init() {
if os.Getenv("MODE") != "statediff" {
fmt.Println("Skipping statediff test")
os.Exit(0)
}
}
func TestPushBlockAndState(t *testing.T) {
conf := DefaultTestConfig
rawURL := os.Getenv(TEST_RAW_URL)
if rawURL == "" {
fmt.Printf("Warning: no raw url configured for statediffing mainnet tests, will look for local file and"+
"then try default endpoint (%s)\r\n", DefaultTestConfig.RawURL)
} else {
conf.RawURL = rawURL
}
for _, blockNumber := range problemBlocks {
conf.BlockNumber = big.NewInt(blockNumber)
tb, trs, err := TestBlockAndReceipts(conf)
require.NoError(t, err)
testPushBlockAndState(t, tb, trs)
}
testBlock, testReceipts, err := TestBlockAndReceiptsFromEnv(conf)
require.NoError(t, err)
testPushBlockAndState(t, testBlock, testReceipts)
}
func testPushBlockAndState(t *testing.T, block *types.Block, receipts types.Receipts) {
t.Run("Test PushBlock and PushStateNode", func(t *testing.T) {
setup(t, block, receipts)
tearDown(t)
})
}
func setup(t *testing.T, testBlock *types.Block, testReceipts types.Receipts) {
db, err = shared.SetupDB()
if err != nil {
t.Fatal(err)
}
ind, err := indexer.NewStateDiffIndexer(chainConf, db)
require.NoError(t, err)
var tx *indexer.BlockTx
tx, err = ind.PushBlock(
testBlock,
testReceipts,
testBlock.Difficulty())
require.NoError(t, err)
defer func() {
if err := tx.Close(err); err != nil {
t.Fatal(err)
}
}()
for _, node := range mocks.StateDiffs {
err = ind.PushStateNode(tx, node)
require.NoError(t, err)
}
shared.ExpectEqual(t, tx.BlockNumber, testBlock.Number().Uint64())
}
func tearDown(t *testing.T) {
indexer.TearDownDB(t, db)
err = db.Close()
require.NoError(t, err)
}

BIN
statediff/indexer/mainnet_tests/receipts_12579670.rlp Normal file
View File

Binary file not shown.

BIN
statediff/indexer/mainnet_tests/receipts_12600011.rlp Normal file
View File

Binary file not shown.

BIN
statediff/indexer/mainnet_tests/receipts_12619985.rlp Normal file
View File

Binary file not shown.

BIN
statediff/indexer/mainnet_tests/receipts_12625121.rlp Normal file
View File

Binary file not shown.

BIN
statediff/indexer/mainnet_tests/receipts_12655432.rlp Normal file
View File

Binary file not shown.

BIN
statediff/indexer/mainnet_tests/receipts_12914664.rlp Normal file
View File

Binary file not shown.

235
statediff/indexer/mainnet_tests/test_helpers.go Normal file
View File

@ -0,0 +1,235 @@
// VulcanizeDB
// Copyright © 2021 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package mainnet_tests
import (
"context"
"errors"
"fmt"
"math/big"
"os"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/ethclient"
"github.com/ethereum/go-ethereum/rlp"
)
const (
defaultBlockFilePath = "./block"
defaultReceiptsFilePath = "./receipts"
)
const (
TEST_RAW_URL = "TEST_RAW_URL"
TEST_BLOCK_NUMBER = "TEST_BLOCK_NUMBER"
)
var problemBlocks = []int64{
12600011,
12619985,
12625121,
12655432,
12579670,
12914664,
}
// TestConfig holds configuration params for mainnet tests
type TestConfig struct {
RawURL string
BlockNumber *big.Int
LocalCache bool
}
// DefaultTestConfig is the default TestConfig
var DefaultTestConfig = TestConfig{
RawURL: "http://127.0.0.1:8545",
BlockNumber: big.NewInt(12914664),
LocalCache: true,
}
// TestBlockAndReceiptsFromEnv retrieves the block and receipts using env variables to override default config block number
func TestBlockAndReceiptsFromEnv(conf TestConfig) (*types.Block, types.Receipts, error) {
blockNumberStr := os.Getenv(TEST_BLOCK_NUMBER)
blockNumber, ok := new(big.Int).SetString(blockNumberStr, 10)
if !ok {
fmt.Printf("Warning: no blockNumber configured for statediffing mainnet tests, using default (%d)\r\n",
DefaultTestConfig.BlockNumber)
} else {
conf.BlockNumber = blockNumber
}
return TestBlockAndReceipts(conf)
}
// TestBlockAndReceipts retrieves the block and receipts for the provided test config
// It first tries to load files from the local system before setting up and using an ethclient.Client to pull the data
func TestBlockAndReceipts(conf TestConfig) (*types.Block, types.Receipts, error) {
var cli *ethclient.Client
var err error
var block *types.Block
var receipts types.Receipts
blockFilePath := fmt.Sprintf("%s_%s.rlp", defaultBlockFilePath, conf.BlockNumber.String())
if _, err = os.Stat(blockFilePath); !errors.Is(err, os.ErrNotExist) {
fmt.Printf("local file (%s) found for block %s\n", blockFilePath, conf.BlockNumber.String())
block, err = LoadBlockRLP(blockFilePath)
if err != nil {
fmt.Printf("loading local file (%s) failed (%s), dialing remote client at %s\n", blockFilePath, err.Error(), conf.RawURL)
cli, err = ethclient.Dial(conf.RawURL)
if err != nil {
return nil, nil, err
}
block, err = FetchBlock(cli, conf.BlockNumber)
if err != nil {
return nil, nil, err
}
if conf.LocalCache {
if err := WriteBlockRLP(blockFilePath, block); err != nil {
return nil, nil, err
}
}
}
} else {
fmt.Printf("no local file found for block %s, dialing remote client at %s\n", conf.BlockNumber.String(), conf.RawURL)
cli, err = ethclient.Dial(conf.RawURL)
if err != nil {
return nil, nil, err
}
block, err = FetchBlock(cli, conf.BlockNumber)
if err != nil {
return nil, nil, err
}
if conf.LocalCache {
if err := WriteBlockRLP(blockFilePath, block); err != nil {
return nil, nil, err
}
}
}
receiptsFilePath := fmt.Sprintf("%s_%s.rlp", defaultReceiptsFilePath, conf.BlockNumber.String())
if _, err = os.Stat(receiptsFilePath); !errors.Is(err, os.ErrNotExist) {
fmt.Printf("local file (%s) found for block %s receipts\n", receiptsFilePath, conf.BlockNumber.String())
receipts, err = LoadReceiptsEncoding(receiptsFilePath, len(block.Transactions()))
if err != nil {
fmt.Printf("loading local file (%s) failed (%s), dialing remote client at %s\n", receiptsFilePath, err.Error(), conf.RawURL)
if cli == nil {
cli, err = ethclient.Dial(conf.RawURL)
if err != nil {
return nil, nil, err
}
}
receipts, err = FetchReceipts(cli, block)
if err != nil {
return nil, nil, err
}
if conf.LocalCache {
if err := WriteReceiptsEncoding(receiptsFilePath, block.Number(), receipts); err != nil {
return nil, nil, err
}
}
}
} else {
fmt.Printf("no local file found for block %s receipts, dialing remote client at %s\n", conf.BlockNumber.String(), conf.RawURL)
if cli == nil {
cli, err = ethclient.Dial(conf.RawURL)
if err != nil {
return nil, nil, err
}
}
receipts, err = FetchReceipts(cli, block)
if err != nil {
return nil, nil, err
}
if conf.LocalCache {
if err := WriteReceiptsEncoding(receiptsFilePath, block.Number(), receipts); err != nil {
return nil, nil, err
}
}
}
return block, receipts, nil
}
// FetchBlock fetches the block at the provided height using the ethclient.Client
func FetchBlock(cli *ethclient.Client, blockNumber *big.Int) (*types.Block, error) {
return cli.BlockByNumber(context.Background(), blockNumber)
}
// FetchReceipts fetches the receipts for the provided block using the ethclient.Client
func FetchReceipts(cli *ethclient.Client, block *types.Block) (types.Receipts, error) {
receipts := make(types.Receipts, len(block.Transactions()))
for i, tx := range block.Transactions() {
rct, err := cli.TransactionReceipt(context.Background(), tx.Hash())
if err != nil {
return nil, err
}
receipts[i] = rct
}
return receipts, nil
}
// WriteBlockRLP writes out the RLP encoding of the block to the provided filePath
func WriteBlockRLP(filePath string, block *types.Block) error {
if filePath == "" {
filePath = fmt.Sprintf("%s_%s.rlp", defaultBlockFilePath, block.Number().String())
}
if _, err := os.Stat(filePath); !errors.Is(err, os.ErrNotExist) {
return fmt.Errorf("cannot create file, file (%s) already exists", filePath)
}
file, err := os.Create(filePath)
if err != nil {
return fmt.Errorf("unable to create file (%s), err: %v", filePath, err)
}
fmt.Printf("writing block rlp to file at %s\r\n", filePath)
if err := block.EncodeRLP(file); err != nil {
return err
}
return file.Close()
}
// LoadBlockRLP loads block from the rlp at filePath
func LoadBlockRLP(filePath string) (*types.Block, error) {
blockBytes, err := os.ReadFile(filePath)
if err != nil {
return nil, err
}
block := new(types.Block)
return block, rlp.DecodeBytes(blockBytes, block)
}
// LoadReceiptsEncoding loads receipts from the encoding at filePath
func LoadReceiptsEncoding(filePath string, cap int) (types.Receipts, error) {
rctsBytes, err := os.ReadFile(filePath)
if err != nil {
return nil, err
}
receipts := new(types.Receipts)
return *receipts, rlp.DecodeBytes(rctsBytes, receipts)
}
// WriteReceiptsEncoding writes out the consensus encoding of the receipts to the provided io.WriteCloser
func WriteReceiptsEncoding(filePath string, blockNumber *big.Int, receipts types.Receipts) error {
if filePath == "" {
filePath = fmt.Sprintf("%s_%s.rlp", defaultReceiptsFilePath, blockNumber.String())
}
if _, err := os.Stat(filePath); !errors.Is(err, os.ErrNotExist) {
return fmt.Errorf("cannot create file, file (%s) already exists", filePath)
}
file, err := os.Create(filePath)
if err != nil {
return fmt.Errorf("unable to create file (%s), err: %v", filePath, err)
}
defer file.Close()
fmt.Printf("writing receipts rlp to file at %s\r\n", filePath)
return rlp.Encode(file, receipts)
}

Some files were not shown because too many files have changed in this diff Show More