From f84b908f6f1aac0541ee394b8b8c4f12647f5b07 Mon Sep 17 00:00:00 2001 From: Elizabeth Date: Mon, 28 Jan 2019 15:31:01 -0600 Subject: [PATCH] Statediffing geth * Write state diff to CSV (#2) * port statediff from https://github.com/jpmorganchase/quorum/blob/9b7fd9af8082795eeeb6863d9746f12b82dd5078/statediff/statediff.go; minor fixes * integrating state diff extracting, building, and persisting into geth processes * work towards persisting created statediffs in ipfs; based off github.com/vulcanize/eth-block-extractor * Add a state diff service * Remove diff extractor from blockchain * Update imports * Move statediff on/off check to geth cmd config * Update starting state diff service * Add debugging logs for creating diff * Add statediff extractor and builder tests and small refactoring * Start to write statediff to a CSV * Restructure statediff directory * Pull CSV publishing methods into their own file * Reformatting due to go fmt * Add gomega to vendor dir * Remove testing focuses * Update statediff tests to use golang test pkg instead of ginkgo - builder_test - extractor_test - publisher_test * Use hexutil.Encode instead of deprecated common.ToHex * Remove OldValue from DiffBigInt and DiffUint64 fields * Update builder test * Remove old storage value from updated accounts * Remove old values from created/deleted accounts * Update publisher to account for only storing current account values * Update service loop and fetching previous block * Update testing - remove statediff ginkgo test suite file - move mocks to their own dir * Updates per go fmt * Updates to tests * Pass statediff mode and path in through cli * Return filename from publisher * Remove some duplication in builder * Remove code field from state diff output this is the contract byte code, and it can still be obtained by querying the db by the codeHash * Consolidate acct diff structs for updated & updated/deleted accts * Include block number in csv filename * Clean up error logging * Cleanup formatting, spelling, etc * Address PR comments * Add contract address and storage value to csv * Refactor accumulating account row in csv publisher * Add DiffStorage struct * Add storage key to csv * Address PR comments * Fix publisher to include rows for accounts that don't have store updates * Update builder test after merging in release/1.8 * Update test contract to include storage on contract intialization - so that we're able to test that storage diffing works for created and deleted accounts (not just updated accounts). * Factor out a common trie iterator method in builder * Apply goimports to statediff * Apply gosimple changes to statediff * Gracefully exit geth command(#4) * Statediff for full node (#6) * Open a trie from the in-memory database * Use a node's LeafKey as an identifier instead of the address It was proving difficult to find look the address up from a given path with a full node (sometimes the value wouldn't exist in the disk db). So, instead, for now we are using the node's LeafKey with is a Keccak256 hash of the address, so if we know the address we can figure out which LeafKey it matches up to. * Make sure that statediff has been processed before pruning * Use blockchain stateCache.OpenTrie for storage diffs * Clean up log lines and remove unnecessary fields from builder * Apply go fmt changes * Add a sleep to the blockchain test * Address PR comments * Address PR comments * refactoring/reorganizing packages * refactoring statediff builder and types and adjusted to relay proofs and paths (still need to make this optional) * refactoring state diff service and adding api which allows for streaming state diff payloads over an rpc websocket subscription * make proofs and paths optional + compress service loop into single for loop (may be missing something here) * option to process intermediate nodes * make state diff rlp serializable * cli parameter to limit statediffing to select account addresses + test * review fixes and fixes for issues ran into in integration * review fixes; proper method signature for api; adjust service so that statediff processing is halted/paused until there is at least one subscriber listening for the results * adjust buffering to improve stability; doc.go; fix notifier err handling * relay receipts with the rest of the data + review fixes/changes * rpc method to get statediff at specific block; requires archival node or the block be within the pruning range * review fixes * fixes after rebase * statediff verison meta * fix linter issues * include total difficulty to the payload * fix state diff builder: emit actual leaf nodes instead of value nodes; diff on the leaf not on the value; emit correct path for intermediate nodes * adjust statediff builder tests to changes and extend to test intermediate nodes; golint * add genesis block to test; handle block 0 in StateDiffAt * rlp files for mainnet blocks 0-3, for tests * builder test on mainnet blocks * common.BytesToHash(path) => crypto.Keaccak256(hash) in builder; BytesToHash produces same hash for e.g. []byte{} and []byte{\x00} - prefix \x00 steps are inconsequential to the hash result * complete tests for early mainnet blocks * diff type for representing deleted accounts * fix builder so that we handle account deletions properly and properly diff storage when an account is moved to a new path; update params * remove cli params; moving them to subscriber defined * remove unneeded bc methods * update service and api; statediffing params are now defined by user through api rather than by service provider by cli * update top level tests * add ability to watch specific storage slots (leaf keys) only * comments; explain logic * update mainnet blocks test * update api_test.go * storage leafkey filter test * cleanup chain maker * adjust chain maker for tests to add an empty account in block1 and switch to EIP-158 afterwards (now we just need to generate enough accounts until one causes the empty account to be touched and removed post-EIP-158 so we can simulate and test that process...); also added 2 new blocks where more contract storage is set and old slots are set to zero so they are removed so we can test that * found an account whose creation causes the empty account to be moved to a new path; this should count as 'touching; the empty account and cause it to be removed according to eip-158... but it doesn't * use new contract in unit tests that has self-destruct ability, so we can test eip-158 since simply moving an account to new path doesn't count as 'touchin' it * handle storage deletions * tests for eip-158 account removal and storage value deletions; there is one edge case left to test where we remove 1 account when only two exist such that the remaining account is moved up and replaces the root branch node * finish testing known edge cases * add endpoint to fetch all state and storage nodes at a given blockheight; useful for generating a recent atate cache/snapshot that we can diff forward from rather than needing to collect all diffs from genesis * test for state trie builder * minor changes/fixes * update version meta * if statediffing is on, lock tries in triedb until the statediffing service signals they are done using them * update version meta * fix mock blockchain; golint; bump patch * increase maxRequestContentLength; bump patch * log the sizes of the state objects we are sending * CI build (#20) * CI: run build on PR and on push to master * CI: debug building geth * CI: fix coping file * CI: fix coping file v2 * CI: temporary upload file to release asset * CI: get release upload_url by tag, upload asset to current relase * CI: fix tag name * fix ci build on statediff_at_anyblock-1.9.11 branch * fix publishing assets in release * bump version meta * use context deadline for timeout in eth_call * collect and emit codehash=>code mappings for state objects * subscription endpoint for retrieving all the codehash=>code mappings that exist at provided height * bump version meta * Implement WriteStateDiffAt * Writes state diffs directly to postgres * Adds CLI flags to configure PG * Refactors builder output with callbacks * Copies refactored postgres handling code from ipld-eth-indexer * rename PostgresCIDWriter.{index->upsert}* * less ambiguous * go.mod update * rm unused * cleanup * output code & codehash iteratively * had to rf some types for this * prometheus metrics output * duplicate recent eth-indexer changes * migrations and metrics... * [wip] prom.Init() here? another CLI flag? * cleanup * tidy & DRY * statediff WriteLoop service + CLI flag * [wip] update test mocks * todo - do something meaningful to test write loop * logging * use geth log * port tests to go testing * drop ginkgo/gomega * fix and cleanup tests * fail before defer statement * delete vendor/ dir * unused * bump version meta * fixes after rebase onto 1.9.23 * bump version meta * fix API registration * bump version meta * use golang 1.15.5 version (#34) * bump version meta; add 0.0.11 branch to actions * bump version meta; update github actions workflows * statediff: refactor metrics * Remove redundant statediff/indexer/prom tooling and use existing prometheus integration. * cleanup * "indexer" namespace for metrics * add reporting loop for db metrics * doc * metrics for statediff stats * metrics namespace/subsystem = statediff/{indexer,service} * statediff: use a worker pool (for direct writes) * fix test * fix chain event subscription * log tweaks * func name * unused import * intermediate chain event channel for metrics * cleanup * bump version meta --- .github/workflows/build.yml | 12 + .github/workflows/on-master.yaml | 27 + .github/workflows/publish.yaml | 34 + Dockerfile.amd64 | 7 + cmd/geth/config.go | 60 +- cmd/geth/main.go | 6 + cmd/geth/usage.go | 11 + cmd/utils/flags.go | 58 +- core/blockchain.go | 48 +- eth/backend.go | 1 + eth/config.go | 4 + go.mod | 20 +- go.sum | 189 ++ internal/ethapi/api.go | 7 +- params/version.go | 2 +- rpc/http.go | 2 +- statediff/api.go | 139 + statediff/builder.go | 754 ++++++ statediff/builder_test.go | 2300 +++++++++++++++++ statediff/doc.go | 57 + statediff/helpers.go | 98 + statediff/indexer/helpers.go | 55 + statediff/indexer/indexer.go | 423 +++ statediff/indexer/indexer_test.go | 292 +++ statediff/indexer/ipfs/ipld/eth_account.go | 175 ++ statediff/indexer/ipfs/ipld/eth_header.go | 256 ++ statediff/indexer/ipfs/ipld/eth_parser.go | 97 + statediff/indexer/ipfs/ipld/eth_receipt.go | 199 ++ .../indexer/ipfs/ipld/eth_receipt_trie.go | 152 ++ statediff/indexer/ipfs/ipld/eth_state.go | 114 + statediff/indexer/ipfs/ipld/eth_storage.go | 100 + statediff/indexer/ipfs/ipld/eth_tx.go | 215 ++ statediff/indexer/ipfs/ipld/eth_tx_trie.go | 152 ++ statediff/indexer/ipfs/ipld/shared.go | 126 + statediff/indexer/ipfs/ipld/trie_node.go | 444 ++++ statediff/indexer/ipfs/models.go | 22 + statediff/indexer/metrics.go | 124 + statediff/indexer/mocks/test_data.go | 194 ++ statediff/indexer/models/models.go | 126 + statediff/indexer/node/node.go | 25 + statediff/indexer/postgres/config.go | 59 + statediff/indexer/postgres/errors.go | 53 + statediff/indexer/postgres/postgres.go | 76 + .../indexer/postgres/postgres_suite_test.go | 25 + statediff/indexer/postgres/postgres_test.go | 131 + statediff/indexer/reward.go | 76 + statediff/indexer/shared/chain_type.go | 78 + statediff/indexer/shared/constants.go | 22 + statediff/indexer/shared/data_type.go | 101 + statediff/indexer/shared/functions.go | 124 + statediff/indexer/shared/test_helpers.go | 68 + statediff/indexer/shared/types.go | 42 + statediff/indexer/test_helpers.go | 60 + statediff/indexer/writer.go | 137 + statediff/mainnet_tests/block0_rlp | Bin 0 -> 540 bytes statediff/mainnet_tests/block1_rlp | Bin 0 -> 537 bytes statediff/mainnet_tests/block2_rlp | Bin 0 -> 544 bytes statediff/mainnet_tests/block3_rlp | Bin 0 -> 1079 bytes statediff/mainnet_tests/builder_test.go | 685 +++++ statediff/metrics.go | 54 + statediff/service.go | 627 +++++ statediff/service_test.go | 291 +++ statediff/testhelpers/helpers.go | 124 + statediff/testhelpers/mocks/blockchain.go | 134 + statediff/testhelpers/mocks/builder.go | 67 + statediff/testhelpers/mocks/service.go | 322 +++ statediff/testhelpers/mocks/service_test.go | 238 ++ statediff/testhelpers/test_data.go | 73 + statediff/types.go | 113 + statediff/types/types.go | 61 + trie/encoding.go | 24 +- trie/encoding_test.go | 6 +- trie/iterator.go | 2 +- trie/sync.go | 2 +- 74 files changed, 10967 insertions(+), 35 deletions(-) create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/on-master.yaml create mode 100644 .github/workflows/publish.yaml create mode 100644 Dockerfile.amd64 create mode 100644 statediff/api.go create mode 100644 statediff/builder.go create mode 100644 statediff/builder_test.go create mode 100644 statediff/doc.go create mode 100644 statediff/helpers.go create mode 100644 statediff/indexer/helpers.go create mode 100644 statediff/indexer/indexer.go create mode 100644 statediff/indexer/indexer_test.go create mode 100644 statediff/indexer/ipfs/ipld/eth_account.go create mode 100644 statediff/indexer/ipfs/ipld/eth_header.go create mode 100644 statediff/indexer/ipfs/ipld/eth_parser.go create mode 100644 statediff/indexer/ipfs/ipld/eth_receipt.go create mode 100644 statediff/indexer/ipfs/ipld/eth_receipt_trie.go create mode 100644 statediff/indexer/ipfs/ipld/eth_state.go create mode 100644 statediff/indexer/ipfs/ipld/eth_storage.go create mode 100644 statediff/indexer/ipfs/ipld/eth_tx.go create mode 100644 statediff/indexer/ipfs/ipld/eth_tx_trie.go create mode 100644 statediff/indexer/ipfs/ipld/shared.go create mode 100644 statediff/indexer/ipfs/ipld/trie_node.go create mode 100644 statediff/indexer/ipfs/models.go create mode 100644 statediff/indexer/metrics.go create mode 100644 statediff/indexer/mocks/test_data.go create mode 100644 statediff/indexer/models/models.go create mode 100644 statediff/indexer/node/node.go create mode 100644 statediff/indexer/postgres/config.go create mode 100644 statediff/indexer/postgres/errors.go create mode 100644 statediff/indexer/postgres/postgres.go create mode 100644 statediff/indexer/postgres/postgres_suite_test.go create mode 100644 statediff/indexer/postgres/postgres_test.go create mode 100644 statediff/indexer/reward.go create mode 100644 statediff/indexer/shared/chain_type.go create mode 100644 statediff/indexer/shared/constants.go create mode 100644 statediff/indexer/shared/data_type.go create mode 100644 statediff/indexer/shared/functions.go create mode 100644 statediff/indexer/shared/test_helpers.go create mode 100644 statediff/indexer/shared/types.go create mode 100644 statediff/indexer/test_helpers.go create mode 100644 statediff/indexer/writer.go create mode 100644 statediff/mainnet_tests/block0_rlp create mode 100644 statediff/mainnet_tests/block1_rlp create mode 100644 statediff/mainnet_tests/block2_rlp create mode 100644 statediff/mainnet_tests/block3_rlp create mode 100644 statediff/mainnet_tests/builder_test.go create mode 100644 statediff/metrics.go create mode 100644 statediff/service.go create mode 100644 statediff/service_test.go create mode 100644 statediff/testhelpers/helpers.go create mode 100644 statediff/testhelpers/mocks/blockchain.go create mode 100644 statediff/testhelpers/mocks/builder.go create mode 100644 statediff/testhelpers/mocks/service.go create mode 100644 statediff/testhelpers/mocks/service_test.go create mode 100644 statediff/testhelpers/test_data.go create mode 100644 statediff/types.go create mode 100644 statediff/types/types.go diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000..8eb113c34 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,12 @@ +name: Docker Build + +on: [pull_request] + +jobs: + build: + name: Run docker build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Run docker build + run: docker build -t vulcanize/go-ethereum -f Dockerfile.amd64 . diff --git a/.github/workflows/on-master.yaml b/.github/workflows/on-master.yaml new file mode 100644 index 000000000..e53dc88b5 --- /dev/null +++ b/.github/workflows/on-master.yaml @@ -0,0 +1,27 @@ +name: Docker Build and publish to Github + +on: + push: + branches: + - v1.9.24-statediff + - v1.9.23-statediff + - v1.9.11-statediff + +jobs: + build: + name: Run docker build and publish + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Run docker build + run: docker build -t vulcanize/go-ethereum -f Dockerfile.amd64 . + - name: Get the version + id: vars + run: echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7}) + - name: Tag docker image + run: docker tag vulcanize/go-ethereum docker.pkg.github.com/vulcanize/go-ethereum/go-ethereum:${{steps.vars.outputs.sha}} + - name: Docker Login + run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin + - name: Docker Push + run: docker push docker.pkg.github.com/vulcanize/go-ethereum/go-ethereum:${{steps.vars.outputs.sha}} + diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml new file mode 100644 index 000000000..3fadf7926 --- /dev/null +++ b/.github/workflows/publish.yaml @@ -0,0 +1,34 @@ +name: Publish geth to release +on: + release: + types: [published] +jobs: + push_to_registries: + name: Publish assets to Release + runs-on: ubuntu-latest + steps: + - name: Get the version + id: vars + run: | + echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7}) + - name: Docker Login to Github Registry + run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin + - name: Docker Pull + run: docker pull docker.pkg.github.com/vulcanize/go-ethereum/go-ethereum:${{steps.vars.outputs.sha}} + - name: Copy ethereum binary file + run: docker run --rm --entrypoint cat docker.pkg.github.com/vulcanize/go-ethereum/go-ethereum:${{steps.vars.outputs.sha}} /go-ethereum/build/bin/geth > geth-linux-amd64 + - name: Get release + id: get_release + uses: bruceadams/get-release@v1.2.0 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Upload Release Asset + id: upload-release-asset + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.get_release.outputs.upload_url }} + asset_path: geth-linux-amd64 + asset_name: geth-linux-amd64 + asset_content_type: application/octet-stream diff --git a/Dockerfile.amd64 b/Dockerfile.amd64 new file mode 100644 index 000000000..7a35376c9 --- /dev/null +++ b/Dockerfile.amd64 @@ -0,0 +1,7 @@ +# Build Geth in a stock Go builder container +FROM golang:1.15.5 as builder + +#RUN apk add --no-cache make gcc musl-dev linux-headers git + +ADD . /go-ethereum +RUN cd /go-ethereum && make geth diff --git a/cmd/geth/config.go b/cmd/geth/config.go index bf1fc55b1..dfb0d4aa7 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -25,6 +25,8 @@ import ( "unicode" "gopkg.in/urfave/cli.v1" + "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/statediff" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/eth" @@ -145,6 +147,9 @@ func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) { cfg.Ethstats.URL = ctx.GlobalString(utils.EthStatsURLFlag.Name) } utils.SetShhConfig(ctx, stack) + if ctx.GlobalBool(utils.StateDiffFlag.Name) { + cfg.Eth.Diffing = true + } return stack, cfg } @@ -162,18 +167,67 @@ func checkWhisper(ctx *cli.Context) { func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { stack, cfg := makeConfigNode(ctx) + if cfg.Eth.SyncMode == downloader.LightSync { + return makeLightNode(ctx, stack, cfg) + } + backend := utils.RegisterEthService(stack, &cfg.Eth) checkWhisper(ctx) + + if ctx.GlobalBool(utils.StateDiffFlag.Name) { + var dbParams *statediff.DBParams + if ctx.GlobalIsSet(utils.StateDiffDBFlag.Name) { + dbParams = new(statediff.DBParams) + dbParams.ConnectionURL = ctx.GlobalString(utils.StateDiffDBFlag.Name) + if ctx.GlobalIsSet(utils.StateDiffDBNodeIDFlag.Name) { + dbParams.ID = ctx.GlobalString(utils.StateDiffDBNodeIDFlag.Name) + } else { + utils.Fatalf("Must specify node ID for statediff DB output") + } + if ctx.GlobalIsSet(utils.StateDiffDBClientNameFlag.Name) { + dbParams.ClientName = ctx.GlobalString(utils.StateDiffDBClientNameFlag.Name) + } else { + utils.Fatalf("Must specify client name for statediff DB output") + } + } else { + if ctx.GlobalBool(utils.StateDiffWritingFlag.Name) { + utils.Fatalf("Must pass DB parameters if enabling statediff write loop") + } + } + params := statediff.ServiceParams{ + DBParams: dbParams, + EnableWriteLoop: ctx.GlobalBool(utils.StateDiffWritingFlag.Name), + NumWorkers: ctx.GlobalUint(utils.StateDiffWorkersFlag.Name), + } + utils.RegisterStateDiffService(stack, backend, params) + } + // Configure GraphQL if requested if ctx.GlobalIsSet(utils.GraphQLEnabledFlag.Name) { - utils.RegisterGraphQLService(stack, backend, cfg.Node) + utils.RegisterGraphQLService(stack, backend.APIBackend, cfg.Node) } // Add the Ethereum Stats daemon if requested. if cfg.Ethstats.URL != "" { - utils.RegisterEthStatsService(stack, backend, cfg.Ethstats.URL) + utils.RegisterEthStatsService(stack, backend.APIBackend, cfg.Ethstats.URL) } - return stack, backend + return stack, backend.APIBackend +} + +func makeLightNode(ctx *cli.Context, stack *node.Node, cfg gethConfig) (*node.Node, ethapi.Backend) { + backend := utils.RegisterLesEthService(stack, &cfg.Eth) + + checkWhisper(ctx) + + // Configure GraphQL if requested + if ctx.GlobalIsSet(utils.GraphQLEnabledFlag.Name) { + utils.RegisterGraphQLService(stack, backend.ApiBackend, cfg.Node) + } + // Add the Ethereum Stats daemon if requested. + if cfg.Ethstats.URL != "" { + utils.RegisterEthStatsService(stack, backend.ApiBackend, cfg.Ethstats.URL) + } + return stack, backend.ApiBackend } // dumpConfig is the dumpconfig command. diff --git a/cmd/geth/main.go b/cmd/geth/main.go index e587a5f86..01fedd489 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -158,6 +158,12 @@ var ( utils.GpoMaxGasPriceFlag, utils.EWASMInterpreterFlag, utils.EVMInterpreterFlag, + utils.StateDiffFlag, + utils.StateDiffDBFlag, + utils.StateDiffDBNodeIDFlag, + utils.StateDiffDBClientNameFlag, + utils.StateDiffWritingFlag, + utils.StateDiffWorkersFlag, configFileFlag, } diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 0e70451ed..d059912ec 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -235,6 +235,17 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.LegacyGraphQLPortFlag, }, debug.DeprecatedFlags...), }, + { + Name: "STATE DIFF", + Flags: []cli.Flag{ + utils.StateDiffFlag, + utils.StateDiffDBFlag, + utils.StateDiffDBNodeIDFlag, + utils.StateDiffDBClientNameFlag, + utils.StateDiffWritingFlag, + utils.StateDiffWorkersFlag, + }, + }, { Name: "MISC", Flags: []cli.Flag{ diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 051bdd630..0ea7d203a 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -63,6 +63,8 @@ import ( "github.com/ethereum/go-ethereum/p2p/nat" "github.com/ethereum/go-ethereum/p2p/netutil" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/statediff" + pcsclite "github.com/gballet/go-libpcsclite" "gopkg.in/urfave/cli.v1" ) @@ -726,6 +728,31 @@ var ( Usage: "External EVM configuration (default = built-in interpreter)", Value: "", } + + StateDiffFlag = cli.BoolFlag{ + Name: "statediff", + Usage: "Enables the processing of state diffs between each block", + } + StateDiffDBFlag = cli.StringFlag{ + Name: "statediff.db", + Usage: "PostgreSQL database connection string for writing state diffs", + } + StateDiffDBNodeIDFlag = cli.StringFlag{ + Name: "statediff.dbnodeid", + Usage: "Node ID to use when writing state diffs to database", + } + StateDiffDBClientNameFlag = cli.StringFlag{ + Name: "statediff.dbclientname", + Usage: "Client name to use when writing state diffs to database", + } + StateDiffWritingFlag = cli.BoolFlag{ + Name: "statediff.writing", + Usage: "Activates progressive writing of state diffs to database as new block are synced", + } + StateDiffWorkersFlag = cli.UintFlag{ + Name: "statediff.workers", + Usage: "Number of concurrent workers to use during statediff processing (0 = 1)", + } ) // MakeDataDir retrieves the currently requested data directory, terminating @@ -991,6 +1018,9 @@ func setWS(ctx *cli.Context, cfg *node.Config) { if ctx.GlobalIsSet(WSApiFlag.Name) { cfg.WSModules = SplitAndTrim(ctx.GlobalString(WSApiFlag.Name)) } + if ctx.GlobalBool(StateDiffFlag.Name) { + cfg.WSModules = append(cfg.WSModules, "statediff") + } } // setIPC creates an IPC path configuration from the set command line flags, @@ -1690,14 +1720,8 @@ func SetDNSDiscoveryDefaults(cfg *eth.Config, genesis common.Hash) { } // RegisterEthService adds an Ethereum client to the stack. -func RegisterEthService(stack *node.Node, cfg *eth.Config) ethapi.Backend { - if cfg.SyncMode == downloader.LightSync { - backend, err := les.New(stack, cfg) - if err != nil { - Fatalf("Failed to register the Ethereum service: %v", err) - } - return backend.ApiBackend - } +// RegisterEthService adds an Ethereum client to the stack. +func RegisterEthService(stack *node.Node, cfg *eth.Config) *eth.Ethereum { backend, err := eth.New(stack, cfg) if err != nil { Fatalf("Failed to register the Ethereum service: %v", err) @@ -1708,7 +1732,16 @@ func RegisterEthService(stack *node.Node, cfg *eth.Config) ethapi.Backend { Fatalf("Failed to create the LES server: %v", err) } } - return backend.APIBackend + return backend +} + +// RegisterLesEthService adds an Ethereum les client to the stack. +func RegisterLesEthService(stack *node.Node, cfg *eth.Config) *les.LightEthereum { + backend, err := les.New(stack, cfg) + if err != nil { + Fatalf("Failed to register the Ethereum service: %v", err) + } + return backend } // RegisterEthStatsService configures the Ethereum Stats daemon and adds it to @@ -1726,6 +1759,13 @@ func RegisterGraphQLService(stack *node.Node, backend ethapi.Backend, cfg node.C } } +// RegisterStateDiffService configures and registers a service to stream state diff data over RPC +func RegisterStateDiffService(stack *node.Node, ethServ *eth.Ethereum, params statediff.ServiceParams) { + if err := statediff.New(stack, ethServ, params); err != nil { + Fatalf("Failed to register the Statediff service: %v", err) + } +} + func SetupMetrics(ctx *cli.Context) { if metrics.Enabled { log.Info("Enabling metrics collection") diff --git a/core/blockchain.go b/core/blockchain.go index bc1db49f3..08f8927d8 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -132,6 +132,7 @@ type CacheConfig struct { Preimages bool // Whether to store preimage of trie key to the disk SnapshotWait bool // Wait for snapshot construction on startup. TODO(karalabe): This is a dirty hack for testing, nuke it + StateDiffing bool // Whether or not the statediffing service is running } // defaultCacheConfig are the default caching values if none are specified by the @@ -212,6 +213,10 @@ type BlockChain struct { shouldPreserve func(*types.Block) bool // Function used to determine whether should preserve the given block. terminateInsert func(common.Hash, uint64) bool // Testing hook used to terminate ancient receipt chain insertion. writeLegacyJournal bool // Testing flag used to flush the snapshot journal in legacy format. + + // Locked roots and their mutex + trieLock sync.Mutex + lockedRoots map[common.Hash]bool } // NewBlockChain returns a fully initialised block chain using information @@ -228,7 +233,6 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par txLookupCache, _ := lru.New(txLookupCacheLimit) futureBlocks, _ := lru.New(maxFutureBlocks) badBlocks, _ := lru.New(badBlockLimit) - bc := &BlockChain{ chainConfig: chainConfig, cacheConfig: cacheConfig, @@ -250,6 +254,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par engine: engine, vmConfig: vmConfig, badBlocks: badBlocks, + lockedRoots: make(map[common.Hash]bool), } bc.validator = NewBlockValidator(chainConfig, bc, engine) bc.prefetcher = newStatePrefetcher(chainConfig, bc, engine) @@ -1040,7 +1045,10 @@ func (bc *BlockChain) Stop() { } } for !bc.triegc.Empty() { - triedb.Dereference(bc.triegc.PopItem().(common.Hash)) + pruneRoot := bc.triegc.PopItem().(common.Hash) + if !bc.TrieLocked(pruneRoot) { + triedb.Dereference(pruneRoot) + } } if size, _ := triedb.Size(); size != 0 { log.Error("Dangling trie nodes after full cleanup") @@ -1546,6 +1554,11 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. triedb.Reference(root, common.Hash{}) // metadata reference to keep trie alive bc.triegc.Push(root, -int64(block.NumberU64())) + // If we are statediffing, lock the trie until the statediffing service is done using it + if bc.cacheConfig.StateDiffing { + bc.LockTrie(root) + } + if current := block.NumberU64(); current > TriesInMemory { // If we exceeded our memory allowance, flush matured singleton nodes to disk var ( @@ -1584,7 +1597,11 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. bc.triegc.Push(root, number) break } - triedb.Dereference(root.(common.Hash)) + pruneRoot := root.(common.Hash) + if !bc.TrieLocked(pruneRoot) { + log.Debug("Dereferencing", "root", root.(common.Hash).Hex()) + triedb.Dereference(pruneRoot) + } } } } @@ -2556,3 +2573,28 @@ func (bc *BlockChain) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscript func (bc *BlockChain) SubscribeBlockProcessingEvent(ch chan<- bool) event.Subscription { return bc.scope.Track(bc.blockProcFeed.Subscribe(ch)) } + +// TrieLocked returns whether the trie associated with the provided root is locked for use +func (bc *BlockChain) TrieLocked(root common.Hash) bool { + bc.trieLock.Lock() + locked, ok := bc.lockedRoots[root] + bc.trieLock.Unlock() + if !ok { + return false + } + return locked +} + +// LockTrie prevents dereferencing of the provided root +func (bc *BlockChain) LockTrie(root common.Hash) { + bc.trieLock.Lock() + bc.lockedRoots[root] = true + bc.trieLock.Unlock() +} + +// UnlockTrie allows dereferencing of the provided root- provided it was previously locked +func (bc *BlockChain) UnlockTrie(root common.Hash) { + bc.trieLock.Lock() + bc.lockedRoots[root] = false + bc.trieLock.Unlock() +} diff --git a/eth/backend.go b/eth/backend.go index 03b0b319b..5fcf522c2 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -170,6 +170,7 @@ func New(stack *node.Node, config *Config) (*Ethereum, error) { TrieTimeLimit: config.TrieTimeout, SnapshotLimit: config.SnapshotCache, Preimages: config.Preimages, + StateDiffing: config.Diffing, } ) eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, chainConfig, eth.engine, vmConfig, eth.shouldPreserve, &config.TxLookupLimit) diff --git a/eth/config.go b/eth/config.go index 0d90376d9..b9e6e4bcd 100644 --- a/eth/config.go +++ b/eth/config.go @@ -187,4 +187,8 @@ type Config struct { // CheckpointOracle is the configuration for checkpoint oracle. CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` + + // Signify whether or not we are producing statediffs + // If we are, do not dereference state roots until the statediffing service is done with them + Diffing bool } diff --git a/go.mod b/go.mod index 0f47809ef..6d8dfc901 100644 --- a/go.mod +++ b/go.mod @@ -35,20 +35,30 @@ require ( github.com/holiman/uint256 v1.1.1 github.com/huin/goupnp v1.0.0 github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883 + github.com/ipfs/go-block-format v0.0.2 + github.com/ipfs/go-cid v0.0.7 + github.com/ipfs/go-ipfs-blockstore v1.0.1 + github.com/ipfs/go-ipfs-ds-help v1.0.0 + github.com/ipfs/go-ipld-format v0.2.0 github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e - github.com/julienschmidt/httprouter v1.1.1-0.20170430222011-975b5c4c7c21 + github.com/jmoiron/sqlx v1.2.0 + github.com/julienschmidt/httprouter v1.2.0 github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356 - github.com/kr/pretty v0.1.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect - github.com/mattn/go-colorable v0.1.0 - github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035 + github.com/lib/pq v1.8.0 + github.com/mattn/go-colorable v0.1.1 + github.com/mattn/go-isatty v0.0.5 + github.com/multiformats/go-multihash v0.0.14 github.com/naoina/go-stringutil v0.1.0 // indirect github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c + github.com/onsi/ginkgo v1.14.0 + github.com/onsi/gomega v1.10.1 github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222 github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 - github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150 + github.com/prometheus/client_golang v0.9.3 + github.com/prometheus/tsdb v0.7.1 github.com/rjeczalik/notify v0.9.1 github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00 github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521 // indirect diff --git a/go.sum b/go.sum index d1bd60858..8a06d1129 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,16 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-pipeline-go v0.2.2 h1:6oiIS9yaG6XCCzhgAgKFfIWyo4LLCiDhZot6ltoThhY= github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= @@ -32,9 +45,16 @@ github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847 h1:rtI0fD4oG/8eVokGVPYJEW1F88p1ZNgXiEIs9thEE4A= github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aws/aws-sdk-go v1.25.48 h1:J82DYDGZHOKHdhx6hD24Tm30c2C3GchYGfN0mf9iKUk= github.com/aws/aws-sdk-go v1.25.48/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6 h1:Eey/GGQ/E5Xp1P2Lyx1qj007hLZfbi0+CoVeJruGCtI= github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ= github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= @@ -43,8 +63,14 @@ github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9 h1:J82+/8rub3qSy0HxEnoYD8cs+HDlHWYrqYXe2Vqxluk= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -66,6 +92,8 @@ github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c h1:JHHhtb9XWJrGNMcr github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/fatih/color v1.3.0 h1:YehCCcyeQ6Km0D6+IapqPinWBK6y+0eB5umvZXK9WPs= github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc h1:jtW8jbpkO4YirRSyepBOH8E+2HEw6/hKkBvFPwhUN8c= github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= @@ -74,18 +102,30 @@ github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0 h1:8HUsc87TaSWLKwrnumgC8/YconD2fJQsRJAsWaPg2ic= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-sourcemap/sourcemap v2.1.2+incompatible h1:0b/xya7BKGhXuqFESKM4oIiRo9WOt2ebz7KxfreD6ug= github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -108,10 +148,16 @@ github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa h1:Q75Upo5UN4JbPFURXZ8nLKYUvF85dyFRop/vQ0Rv+64= github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989 h1:giknQ4mEuDFmmHSrGcbargOuLHQGtywqo4mheITex54= github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277 h1:E0whKxgp2ojts0FDgUA8dl62bmH0LxKanMoBr6MDTDM= github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= +github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= +github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/holiman/uint256 v1.1.1 h1:4JywC80b+/hSfljFlEBLHrrh+CIONLDz9NuFl0af4Mw= @@ -123,16 +169,54 @@ github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7 github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883 h1:FSeK4fZCo8u40n2JMnyAsd6x7+SbvoOMHvQOU/n10P4= github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= +github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= +github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= +github.com/ipfs/go-block-format v0.0.2 h1:qPDvcP19izTjU8rgo6p7gTXZlkMkF5bz5G3fqIsSCPE= +github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= +github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= +github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= +github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= +github.com/ipfs/go-cid v0.0.7 h1:ysQJVJA3fNDF1qigJbsSQOdjhVLsOEoPdh0+R97k3jY= +github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= +github.com/ipfs/go-datastore v0.4.1/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= +github.com/ipfs/go-datastore v0.4.2 h1:h8/n7WPzhp239kkLws+epN3Ic7YtcBPgcaXfEfdVDWM= +github.com/ipfs/go-datastore v0.4.2/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= +github.com/ipfs/go-ipfs-blockstore v1.0.1 h1:fnuVj4XdZp4yExhd0CnUwAiMNJHiPnfInhiuwz4lW1w= +github.com/ipfs/go-ipfs-blockstore v1.0.1/go.mod h1:MGNZlHNEnR4KGgPHM3/k8lBySIOK2Ve+0KjZubKlaOE= +github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= +github.com/ipfs/go-ipfs-ds-help v1.0.0 h1:bEQ8hMGs80h0sR8O4tfDgV6B01aaF9qeTrujrTLYV3g= +github.com/ipfs/go-ipfs-ds-help v1.0.0/go.mod h1:ujAbkeIgkKAWtxxNkoZHWLCyk5JpPoKnGyCcsoF6ueE= +github.com/ipfs/go-ipfs-util v0.0.1 h1:Wz9bL2wB2YBJqggkA4dD7oSmqB4cAnpNbGrlHJulv50= +github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= +github.com/ipfs/go-ipld-format v0.2.0 h1:xGlJKkArkmBvowr+GMCX0FEZtkro71K1AwiKnL37mwA= +github.com/ipfs/go-ipld-format v0.2.0/go.mod h1:3l3C1uKoadTPbeNfrDi+xMInYKlx2Cvg1BuydPSdzQs= +github.com/ipfs/go-log v0.0.1 h1:9XTUN/rW64BCG1YhPK9Hoy3q8nr4gOmHHBpgFdfw6Lc= +github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= +github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= +github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 h1:6OvNmYgJyexcZ3pYbTI9jWx5tHo1Dee/tWbLMfPe2TA= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= +github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8 h1:bspPhN+oKYFk5fcGNuQzp6IGzYQSenLEgH3s6jkXrWw= +github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY= github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e h1:UvSe12bq+Uj2hWd8aOlwPmoZ+CITRFrdit+sDGfAg8U= github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= +github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.1.1-0.20170430222011-975b5c4c7c21 h1:F/iKcka0K2LgnKy/fgSBf235AETtm1n1TvBzqu40LE0= github.com/julienschmidt/httprouter v1.1.1-0.20170430222011-975b5c4c7c21/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356 h1:I/yrLt2WilKxlQKCM52clh5rGzTKpVctGT1lH4Dc8Jw= github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= @@ -142,17 +226,64 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg= +github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM= +github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.0 h1:v2XXALHHh6zHfYTJ+cSkwtyffnaOyR1MXaA91mTrb8o= github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d h1:oNAwILwmgWKFpuU+dXvI6dl9jG2mAWAZLX3r9s0PPiw= github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035 h1:USWjF42jDCSEeikX/G1g40ZWnsPXN5WkZ4jMHZWyBK4= github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.5 h1:tHXDdz1cpzGaovsTB+TVB8q90WEokoVmfMqoVcrLUgw= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g= +github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= +github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= +github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771 h1:MHkK1uRtFbVqvAgvWxafZe54+5uBxLluGylDiKgdhwo= +github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= +github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc= +github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI= +github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= +github.com/multiformats/go-base36 v0.1.0 h1:JR6TyF7JjGd3m6FbLU2cOxhC0Li8z8dLNGQ89tUg4F4= +github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM= +github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= +github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk= +github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= +github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= +github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= +github.com/multiformats/go-multihash v0.0.14 h1:QoBceQYQQtNUuf6s7wHxnE2c8bhbMqhfGzNI032se/I= +github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= +github.com/multiformats/go-varint v0.0.5 h1:XVZwSo04Cs3j/jS0uAEPpT3JY6DzMcVLLoWOSnCxOjg= +github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hzifhks= github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 h1:shk/vn9oCoOTmwcouEdwIeOtOGA/ELRUw/GwvxwfT+0= @@ -164,16 +295,22 @@ github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXW github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c h1:1RHs3tNxjXGHeul8z2t6H2N2TlAqpKe5yryJztRx4Jk= github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222 h1:goeTyGkArOZIVOMA0dQbyuPWGNQJZGPwPu/QS9GlpnA= github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= +github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 h1:oYW+YCJ1pachXTQmzR3rNLYGGz4g/UgFcjb28p/viDM= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -181,14 +318,27 @@ github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3 h1:9iH4JKXLzFbOAdtqv/a+j8aewx2Y8lAjAydhbaScPF8= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150 h1:ZeU+auZj1iNzN8iVhff6M38Mfu73FQiJve/GEXYJBjE= github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rjeczalik/notify v0.9.1 h1:CLCKso/QK1snAlnhNR/CNvNiFU2saUtjV0bx3EwNeCE= github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00 h1:8DPul/X0IT/1TNMIxoKLwdemEOBBHDC/K4EB16Cw5WE= github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521 h1:3hxavr+IHMsQBrYUPQM5v0CgENFktkkbg1sfpgM3h20= @@ -197,7 +347,15 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/shirou/gopsutil v2.20.5+incompatible h1:tYH07UPoQt0OCQdgWWMgYHy3/a9bcxNpBIysykNIP7I= github.com/shirou/gopsutil v2.20.5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 h1:Gb2Tyox57NRNuZ2d3rmvB3pcmbu7O1RS3m8WRx7ilrg= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570 h1:gIlAHnH1vJb5vwEjIp5kBj/eu99p/bl0Ay2goiPe5xE= @@ -205,6 +363,7 @@ github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570/go.mod h1:8 github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3 h1:njlZPzLwU639dk2kqnCPPv+wNjq7Xb6EfUxe/oX0/NM= github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3/go.mod h1:hpGUWaI9xL8pRQCTXQgocU38Qw1g0Us7n5PxxTwTCYU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= @@ -214,10 +373,17 @@ github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca/go.mod h1:u2MKk github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef h1:wHSqTBrZW24CsNJDfeh9Ex6Pm0Rcpc7qrgKBiL44vF4= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc h1:9lDbC6Rz4bwmou+oE6Dt4Cb2BGMur5eR/GYptkKUVHo= +github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208 h1:1cngl9mPEoITZG8s8cVcUy5CeIBYhEESkOB7m6Gmkrk= github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+mw0EzQ08zFqg7pK3FebNXpaMsRy2RT+Ees= +go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= @@ -225,6 +391,7 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20200801112145-973feb4309de h1:OVJ6QQUBAesB8CZijKDSsXX7xYVtUhrkY0gwMfbi4p4= golang.org/x/mobile v0.0.0-20200801112145-973feb4309de/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4= @@ -234,6 +401,8 @@ golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd h1:ePuNC7PZ6O5BzgPn9bZayER golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -244,10 +413,16 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -259,14 +434,18 @@ golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8 h1:AvbQYmiaaaza3cW3QXRyPo5kYgpFIzOAfeAAN7m3qQ4= golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69 h1:yBHHx+XZqXJBm6Exke3N7V9gnlsyXxoCPEb1yVenjfk= golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -287,8 +466,12 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6 h1:a6cXbcDDUkSBlpnkWV1bJ+vv3mOgQEltEJ2rPxroVu0= @@ -297,6 +480,7 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0= gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -304,3 +488,8 @@ gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 030bdb37a..4aa91bef2 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -932,7 +932,12 @@ func (s *PublicBlockChainAPI) Call(ctx context.Context, args CallArgs, blockNrOr if overrides != nil { accounts = *overrides } - result, err := DoCall(ctx, s.b, args, blockNrOrHash, accounts, vm.Config{}, 5*time.Second, s.b.RPCGasCap()) + timeout := 5 * time.Second + d, ok := ctx.Deadline() + if ok { + timeout = d.Sub(time.Now()) + } + result, err := DoCall(ctx, s.b, args, blockNrOrHash, accounts, vm.Config{}, timeout, s.b.RPCGasCap()) if err != nil { return nil, err } diff --git a/params/version.go b/params/version.go index 3237b51f8..3c7f6e7cb 100644 --- a/params/version.go +++ b/params/version.go @@ -24,7 +24,7 @@ const ( VersionMajor = 1 // Major version component of the current release VersionMinor = 9 // Minor version component of the current release VersionPatch = 25 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMeta = "statediff-0.0.13" // Version metadata to append to the version string ) // Version holds the textual version string. diff --git a/rpc/http.go b/rpc/http.go index 87a96e49e..ddbbf6fe4 100644 --- a/rpc/http.go +++ b/rpc/http.go @@ -32,7 +32,7 @@ import ( ) const ( - maxRequestContentLength = 1024 * 1024 * 5 + maxRequestContentLength = 1024 * 1024 * 12 contentType = "application/json" ) diff --git a/statediff/api.go b/statediff/api.go new file mode 100644 index 000000000..33ecedd67 --- /dev/null +++ b/statediff/api.go @@ -0,0 +1,139 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package statediff + +import ( + "context" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rpc" + . "github.com/ethereum/go-ethereum/statediff/types" +) + +// APIName is the namespace used for the state diffing service API +const APIName = "statediff" + +// APIVersion is the version of the state diffing service API +const APIVersion = "0.0.1" + +// PublicStateDiffAPI provides an RPC subscription interface +// that can be used to stream out state diffs as they +// are produced by a full node +type PublicStateDiffAPI struct { + sds IService +} + +// NewPublicStateDiffAPI creates an rpc subscription interface for the underlying statediff service +func NewPublicStateDiffAPI(sds IService) *PublicStateDiffAPI { + return &PublicStateDiffAPI{ + sds: sds, + } +} + +// Stream is the public method to setup a subscription that fires off statediff service payloads as they are created +func (api *PublicStateDiffAPI) Stream(ctx context.Context, params Params) (*rpc.Subscription, error) { + // ensure that the RPC connection supports subscriptions + notifier, supported := rpc.NotifierFromContext(ctx) + if !supported { + return nil, rpc.ErrNotificationsUnsupported + } + + // create subscription and start waiting for events + rpcSub := notifier.CreateSubscription() + + go func() { + // subscribe to events from the statediff service + payloadChannel := make(chan Payload, chainEventChanSize) + quitChan := make(chan bool, 1) + api.sds.Subscribe(rpcSub.ID, payloadChannel, quitChan, params) + // loop and await payloads and relay them to the subscriber with the notifier + for { + select { + case payload := <-payloadChannel: + if err := notifier.Notify(rpcSub.ID, payload); err != nil { + log.Error("Failed to send state diff packet; error: " + err.Error()) + if err := api.sds.Unsubscribe(rpcSub.ID); err != nil { + log.Error("Failed to unsubscribe from the state diff service; error: " + err.Error()) + } + return + } + case err := <-rpcSub.Err(): + if err != nil { + log.Error("State diff service rpcSub error: " + err.Error()) + err = api.sds.Unsubscribe(rpcSub.ID) + if err != nil { + log.Error("Failed to unsubscribe from the state diff service; error: " + err.Error()) + } + return + } + case <-quitChan: + // don't need to unsubscribe, service does so before sending the quit signal + return + } + } + }() + + return rpcSub, nil +} + +// StateDiffAt returns a state diff payload at the specific blockheight +func (api *PublicStateDiffAPI) StateDiffAt(ctx context.Context, blockNumber uint64, params Params) (*Payload, error) { + return api.sds.StateDiffAt(blockNumber, params) +} + +// StateTrieAt returns a state trie payload at the specific blockheight +func (api *PublicStateDiffAPI) StateTrieAt(ctx context.Context, blockNumber uint64, params Params) (*Payload, error) { + return api.sds.StateTrieAt(blockNumber, params) +} + +// StreamCodeAndCodeHash writes all of the codehash=>code pairs out to a websocket channel +func (api *PublicStateDiffAPI) StreamCodeAndCodeHash(ctx context.Context, blockNumber uint64) (*rpc.Subscription, error) { + // ensure that the RPC connection supports subscriptions + notifier, supported := rpc.NotifierFromContext(ctx) + if !supported { + return nil, rpc.ErrNotificationsUnsupported + } + + // create subscription and start waiting for events + rpcSub := notifier.CreateSubscription() + payloadChan := make(chan CodeAndCodeHash, chainEventChanSize) + quitChan := make(chan bool) + api.sds.StreamCodeAndCodeHash(blockNumber, payloadChan, quitChan) + go func() { + for { + select { + case payload := <-payloadChan: + if err := notifier.Notify(rpcSub.ID, payload); err != nil { + log.Error("Failed to send code and codehash packet", "err", err) + return + } + case err := <-rpcSub.Err(): + log.Error("State diff service rpcSub error", "err", err) + return + case <-quitChan: + return + } + } + }() + + return rpcSub, nil +} + +// WriteStateDiffAt writes a state diff object directly to DB at the specific blockheight +func (api *PublicStateDiffAPI) WriteStateDiffAt(ctx context.Context, blockNumber uint64, params Params) error { + return api.sds.WriteStateDiffAt(blockNumber, params) +} diff --git a/statediff/builder.go b/statediff/builder.go new file mode 100644 index 000000000..c9757b794 --- /dev/null +++ b/statediff/builder.go @@ -0,0 +1,754 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package statediff + +import ( + "bytes" + "fmt" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" + . "github.com/ethereum/go-ethereum/statediff/types" + "github.com/ethereum/go-ethereum/trie" +) + +var ( + nullHashBytes = common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000000") + emptyNode, _ = rlp.EncodeToBytes([]byte{}) + emptyContractRoot = crypto.Keccak256Hash(emptyNode) + nullCodeHash = crypto.Keccak256Hash([]byte{}).Bytes() +) + +// Builder interface exposes the method for building a state diff between two blocks +type Builder interface { + BuildStateDiffObject(args Args, params Params) (StateObject, error) + BuildStateTrieObject(current *types.Block) (StateObject, error) + WriteStateDiffObject(args StateRoots, params Params, output StateNodeSink, codeOutput CodeSink) error +} + +type builder struct { + stateCache state.Database +} + +func resolveNode(it trie.NodeIterator, trieDB *trie.Database) (StateNode, []interface{}, error) { + nodePath := make([]byte, len(it.Path())) + copy(nodePath, it.Path()) + node, err := trieDB.Node(it.Hash()) + if err != nil { + return StateNode{}, nil, err + } + var nodeElements []interface{} + if err := rlp.DecodeBytes(node, &nodeElements); err != nil { + return StateNode{}, nil, err + } + ty, err := CheckKeyType(nodeElements) + if err != nil { + return StateNode{}, nil, err + } + return StateNode{ + NodeType: ty, + Path: nodePath, + NodeValue: node, + }, nodeElements, nil +} + +// convenience +func stateNodeAppender(nodes *[]StateNode) StateNodeSink { + return func(node StateNode) error { + *nodes = append(*nodes, node) + return nil + } +} +func storageNodeAppender(nodes *[]StorageNode) StorageNodeSink { + return func(node StorageNode) error { + *nodes = append(*nodes, node) + return nil + } +} +func codeMappingAppender(codeAndCodeHashes *[]CodeAndCodeHash) CodeSink { + return func(c CodeAndCodeHash) error { + *codeAndCodeHashes = append(*codeAndCodeHashes, c) + return nil + } +} + +// NewBuilder is used to create a statediff builder +func NewBuilder(stateCache state.Database) Builder { + return &builder{ + stateCache: stateCache, // state cache is safe for concurrent reads + } +} + +// BuildStateTrieObject builds a state trie object from the provided block +func (sdb *builder) BuildStateTrieObject(current *types.Block) (StateObject, error) { + currentTrie, err := sdb.stateCache.OpenTrie(current.Root()) + if err != nil { + return StateObject{}, fmt.Errorf("error creating trie for block %d: %v", current.Number(), err) + } + it := currentTrie.NodeIterator([]byte{}) + stateNodes, codeAndCodeHashes, err := sdb.buildStateTrie(it) + if err != nil { + return StateObject{}, fmt.Errorf("error collecting state nodes for block %d: %v", current.Number(), err) + } + return StateObject{ + BlockNumber: current.Number(), + BlockHash: current.Hash(), + Nodes: stateNodes, + CodeAndCodeHashes: codeAndCodeHashes, + }, nil +} + +func (sdb *builder) buildStateTrie(it trie.NodeIterator) ([]StateNode, []CodeAndCodeHash, error) { + stateNodes := make([]StateNode, 0) + codeAndCodeHashes := make([]CodeAndCodeHash, 0) + for it.Next(true) { + // skip value nodes + if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) { + continue + } + node, nodeElements, err := resolveNode(it, sdb.stateCache.TrieDB()) + if err != nil { + return nil, nil, err + } + switch node.NodeType { + case Leaf: + var account state.Account + if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil { + return nil, nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", node.Path, err) + } + partialPath := trie.CompactToHex(nodeElements[0].([]byte)) + valueNodePath := append(node.Path, partialPath...) + encodedPath := trie.HexToCompact(valueNodePath) + leafKey := encodedPath[1:] + node.LeafKey = leafKey + if !bytes.Equal(account.CodeHash, nullCodeHash) { + var storageNodes []StorageNode + err := sdb.buildStorageNodesEventual(account.Root, nil, true, storageNodeAppender(&storageNodes)) + if err != nil { + return nil, nil, fmt.Errorf("failed building eventual storage diffs for account %+v\r\nerror: %v", account, err) + } + node.StorageNodes = storageNodes + // emit codehash => code mappings for cod + codeHash := common.BytesToHash(account.CodeHash) + code, err := sdb.stateCache.ContractCode(common.Hash{}, codeHash) + if err != nil { + return nil, nil, fmt.Errorf("failed to retrieve code for codehash %s\r\n error: %v", codeHash.String(), err) + } + codeAndCodeHashes = append(codeAndCodeHashes, CodeAndCodeHash{ + Hash: codeHash, + Code: code, + }) + } + stateNodes = append(stateNodes, node) + case Extension, Branch: + stateNodes = append(stateNodes, node) + default: + return nil, nil, fmt.Errorf("unexpected node type %s", node.NodeType) + } + } + return stateNodes, codeAndCodeHashes, it.Error() +} + +// BuildStateDiffObject builds a statediff object from two blocks and the provided parameters +func (sdb *builder) BuildStateDiffObject(args Args, params Params) (StateObject, error) { + var stateNodes []StateNode + var codeAndCodeHashes []CodeAndCodeHash + err := sdb.WriteStateDiffObject( + StateRoots{OldStateRoot: args.OldStateRoot, NewStateRoot: args.NewStateRoot}, + params, stateNodeAppender(&stateNodes), codeMappingAppender(&codeAndCodeHashes)) + if err != nil { + return StateObject{}, err + } + return StateObject{ + BlockHash: args.BlockHash, + BlockNumber: args.BlockNumber, + Nodes: stateNodes, + CodeAndCodeHashes: codeAndCodeHashes, + }, nil +} + +// Writes a statediff object to output callback +func (sdb *builder) WriteStateDiffObject(args StateRoots, params Params, output StateNodeSink, codeOutput CodeSink) error { + if !params.IntermediateStateNodes || len(params.WatchedAddresses) > 0 { + // if we are watching only specific accounts then we are only diffing leaf nodes + return sdb.buildStateDiffWithoutIntermediateStateNodes(args, params, output, codeOutput) + } else { + return sdb.buildStateDiffWithIntermediateStateNodes(args, params, output, codeOutput) + } +} + +func (sdb *builder) buildStateDiffWithIntermediateStateNodes(args StateRoots, params Params, output StateNodeSink, codeOutput CodeSink) error { + // Load tries for old and new states + oldTrie, err := sdb.stateCache.OpenTrie(args.OldStateRoot) + if err != nil { + return fmt.Errorf("error creating trie for oldStateRoot: %v", err) + } + newTrie, err := sdb.stateCache.OpenTrie(args.NewStateRoot) + if err != nil { + return fmt.Errorf("error creating trie for newStateRoot: %v", err) + } + + // collect a slice of all the intermediate nodes that were touched and exist at B + // a map of their leafkey to all the accounts that were touched and exist at B + // and a slice of all the paths for the nodes in both of the above sets + diffAccountsAtB, diffPathsAtB, err := sdb.createdAndUpdatedStateWithIntermediateNodes( + oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), + output) + if err != nil { + return fmt.Errorf("error collecting createdAndUpdatedNodes: %v", err) + } + + // collect a slice of all the nodes that existed at a path in A that doesn't exist in B + // a map of their leafkey to all the accounts that were touched and exist at A + diffAccountsAtA, err := sdb.deletedOrUpdatedState( + oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), + diffPathsAtB, output) + if err != nil { + return fmt.Errorf("error collecting deletedOrUpdatedNodes: %v", err) + } + + // collect and sort the leafkey keys for both account mappings into a slice + createKeys := sortKeys(diffAccountsAtB) + deleteKeys := sortKeys(diffAccountsAtA) + + // and then find the intersection of these keys + // these are the leafkeys for the accounts which exist at both A and B but are different + // this also mutates the passed in createKeys and deleteKeys, removing the intersection keys + // and leaving the truly created or deleted keys in place + updatedKeys := findIntersection(createKeys, deleteKeys) + + // build the diff nodes for the updated accounts using the mappings at both A and B as directed by the keys found as the intersection of the two + err = sdb.buildAccountUpdates( + diffAccountsAtB, diffAccountsAtA, updatedKeys, + params.WatchedStorageSlots, params.IntermediateStorageNodes, output) + if err != nil { + return fmt.Errorf("error building diff for updated accounts: %v", err) + } + // build the diff nodes for created accounts + err = sdb.buildAccountCreations(diffAccountsAtB, params.WatchedStorageSlots, params.IntermediateStorageNodes, output, codeOutput) + if err != nil { + return fmt.Errorf("error building diff for created accounts: %v", err) + } + return nil +} + +func (sdb *builder) buildStateDiffWithoutIntermediateStateNodes(args StateRoots, params Params, output StateNodeSink, codeOutput CodeSink) error { + // Load tries for old (A) and new (B) states + oldTrie, err := sdb.stateCache.OpenTrie(args.OldStateRoot) + if err != nil { + return fmt.Errorf("error creating trie for oldStateRoot: %v", err) + } + newTrie, err := sdb.stateCache.OpenTrie(args.NewStateRoot) + if err != nil { + return fmt.Errorf("error creating trie for newStateRoot: %v", err) + } + + // collect a map of their leafkey to all the accounts that were touched and exist at B + // and a slice of all the paths for the nodes in both of the above sets + diffAccountsAtB, diffPathsAtB, err := sdb.createdAndUpdatedState( + oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), + params.WatchedAddresses) + if err != nil { + return fmt.Errorf("error collecting createdAndUpdatedNodes: %v", err) + } + + // collect a slice of all the nodes that existed at a path in A that doesn't exist in B + // a map of their leafkey to all the accounts that were touched and exist at A + diffAccountsAtA, err := sdb.deletedOrUpdatedState( + oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), + diffPathsAtB, output) + if err != nil { + return fmt.Errorf("error collecting deletedOrUpdatedNodes: %v", err) + } + + // collect and sort the leafkeys for both account mappings into a slice + createKeys := sortKeys(diffAccountsAtB) + deleteKeys := sortKeys(diffAccountsAtA) + + // and then find the intersection of these keys + // these are the leafkeys for the accounts which exist at both A and B but are different + // this also mutates the passed in createKeys and deleteKeys, removing in intersection keys + // and leaving the truly created or deleted keys in place + updatedKeys := findIntersection(createKeys, deleteKeys) + + // build the diff nodes for the updated accounts using the mappings at both A and B as directed by the keys found as the intersection of the two + err = sdb.buildAccountUpdates( + diffAccountsAtB, diffAccountsAtA, updatedKeys, + params.WatchedStorageSlots, params.IntermediateStorageNodes, output) + if err != nil { + return fmt.Errorf("error building diff for updated accounts: %v", err) + } + // build the diff nodes for created accounts + err = sdb.buildAccountCreations(diffAccountsAtB, params.WatchedStorageSlots, params.IntermediateStorageNodes, output, codeOutput) + if err != nil { + return fmt.Errorf("error building diff for created accounts: %v", err) + } + return nil +} + +// createdAndUpdatedState returns +// a mapping of their leafkeys to all the accounts that exist in a different state at B than A +// and a slice of the paths for all of the nodes included in both +func (sdb *builder) createdAndUpdatedState(a, b trie.NodeIterator, watchedAddresses []common.Address) (AccountMap, map[string]bool, error) { + diffPathsAtB := make(map[string]bool) + diffAcountsAtB := make(AccountMap) + it, _ := trie.NewDifferenceIterator(a, b) + for it.Next(true) { + // skip value nodes + if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) { + continue + } + node, nodeElements, err := resolveNode(it, sdb.stateCache.TrieDB()) + if err != nil { + return nil, nil, err + } + if node.NodeType == Leaf { + // created vs updated is important for leaf nodes since we need to diff their storage + // so we need to map all changed accounts at B to their leafkey, since account can change pathes but not leafkey + var account state.Account + if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil { + return nil, nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", node.Path, err) + } + partialPath := trie.CompactToHex(nodeElements[0].([]byte)) + valueNodePath := append(node.Path, partialPath...) + encodedPath := trie.HexToCompact(valueNodePath) + leafKey := encodedPath[1:] + if isWatchedAddress(watchedAddresses, leafKey) { + diffAcountsAtB[common.Bytes2Hex(leafKey)] = accountWrapper{ + NodeType: node.NodeType, + Path: node.Path, + NodeValue: node.NodeValue, + LeafKey: leafKey, + Account: &account, + } + } + } + // add both intermediate and leaf node paths to the list of diffPathsAtB + diffPathsAtB[common.Bytes2Hex(node.Path)] = true + } + return diffAcountsAtB, diffPathsAtB, it.Error() +} + +// createdAndUpdatedStateWithIntermediateNodes returns +// a slice of all the intermediate nodes that exist in a different state at B than A +// a mapping of their leafkeys to all the accounts that exist in a different state at B than A +// and a slice of the paths for all of the nodes included in both +func (sdb *builder) createdAndUpdatedStateWithIntermediateNodes(a, b trie.NodeIterator, output StateNodeSink) (AccountMap, map[string]bool, error) { + diffPathsAtB := make(map[string]bool) + diffAcountsAtB := make(AccountMap) + it, _ := trie.NewDifferenceIterator(a, b) + for it.Next(true) { + // skip value nodes + if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) { + continue + } + node, nodeElements, err := resolveNode(it, sdb.stateCache.TrieDB()) + if err != nil { + return nil, nil, err + } + switch node.NodeType { + case Leaf: + // created vs updated is important for leaf nodes since we need to diff their storage + // so we need to map all changed accounts at B to their leafkey, since account can change paths but not leafkey + var account state.Account + if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil { + return nil, nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", node.Path, err) + } + partialPath := trie.CompactToHex(nodeElements[0].([]byte)) + valueNodePath := append(node.Path, partialPath...) + encodedPath := trie.HexToCompact(valueNodePath) + leafKey := encodedPath[1:] + diffAcountsAtB[common.Bytes2Hex(leafKey)] = accountWrapper{ + NodeType: node.NodeType, + Path: node.Path, + NodeValue: node.NodeValue, + LeafKey: leafKey, + Account: &account, + } + case Extension, Branch: + // create a diff for any intermediate node that has changed at b + // created vs updated makes no difference for intermediate nodes since we do not need to diff storage + if err := output(StateNode{ + NodeType: node.NodeType, + Path: node.Path, + NodeValue: node.NodeValue, + }); err != nil { + return nil, nil, err + } + default: + return nil, nil, fmt.Errorf("unexpected node type %s", node.NodeType) + } + // add both intermediate and leaf node paths to the list of diffPathsAtB + diffPathsAtB[common.Bytes2Hex(node.Path)] = true + } + return diffAcountsAtB, diffPathsAtB, it.Error() +} + +// deletedOrUpdatedState returns a slice of all the pathes that are emptied at B +// and a mapping of their leafkeys to all the accounts that exist in a different state at A than B +func (sdb *builder) deletedOrUpdatedState(a, b trie.NodeIterator, diffPathsAtB map[string]bool, output StateNodeSink) (AccountMap, error) { + diffAccountAtA := make(AccountMap) + it, _ := trie.NewDifferenceIterator(b, a) + for it.Next(true) { + // skip value nodes + if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) { + continue + } + node, nodeElements, err := resolveNode(it, sdb.stateCache.TrieDB()) + if err != nil { + return nil, err + } + // if this node's path did not show up in diffPathsAtB + // that means the node at this path was deleted (or moved) in B + // emit an empty "removed" diff to signify as such + if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; !ok { + if err := output(StateNode{ + Path: node.Path, + NodeValue: []byte{}, + NodeType: Removed, + }); err != nil { + return nil, err + } + } + switch node.NodeType { + case Leaf: + // map all different accounts at A to their leafkey + var account state.Account + if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil { + return nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", node.Path, err) + } + partialPath := trie.CompactToHex(nodeElements[0].([]byte)) + valueNodePath := append(node.Path, partialPath...) + encodedPath := trie.HexToCompact(valueNodePath) + leafKey := encodedPath[1:] + diffAccountAtA[common.Bytes2Hex(leafKey)] = accountWrapper{ + NodeType: node.NodeType, + Path: node.Path, + NodeValue: node.NodeValue, + LeafKey: leafKey, + Account: &account, + } + case Extension, Branch: + // fall through, we did everything we need to do with these node types + default: + return nil, fmt.Errorf("unexpected node type %s", node.NodeType) + } + } + return diffAccountAtA, it.Error() +} + +// buildAccountUpdates uses the account diffs maps for A => B and B => A and the known intersection of their leafkeys +// to generate the statediff node objects for all of the accounts that existed at both A and B but in different states +// needs to be called before building account creations and deletions as this mutates +// those account maps to remove the accounts which were updated +func (sdb *builder) buildAccountUpdates(creations, deletions AccountMap, updatedKeys []string, + watchedStorageKeys []common.Hash, intermediateStorageNodes bool, output StateNodeSink) error { + var err error + for _, key := range updatedKeys { + createdAcc := creations[key] + deletedAcc := deletions[key] + var storageDiffs []StorageNode + if deletedAcc.Account != nil && createdAcc.Account != nil { + oldSR := deletedAcc.Account.Root + newSR := createdAcc.Account.Root + err = sdb.buildStorageNodesIncremental( + oldSR, newSR, watchedStorageKeys, intermediateStorageNodes, + storageNodeAppender(&storageDiffs)) + if err != nil { + return fmt.Errorf("failed building incremental storage diffs for account with leafkey %s\r\nerror: %v", key, err) + } + } + if err = output(StateNode{ + NodeType: createdAcc.NodeType, + Path: createdAcc.Path, + NodeValue: createdAcc.NodeValue, + LeafKey: createdAcc.LeafKey, + StorageNodes: storageDiffs, + }); err != nil { + return err + } + delete(creations, key) + delete(deletions, key) + } + + return nil +} + +// buildAccountCreations returns the statediff node objects for all the accounts that exist at B but not at A +// it also returns the code and codehash for created contract accounts +func (sdb *builder) buildAccountCreations(accounts AccountMap, watchedStorageKeys []common.Hash, intermediateStorageNodes bool, output StateNodeSink, codeOutput CodeSink) error { + for _, val := range accounts { + diff := StateNode{ + NodeType: val.NodeType, + Path: val.Path, + LeafKey: val.LeafKey, + NodeValue: val.NodeValue, + } + if !bytes.Equal(val.Account.CodeHash, nullCodeHash) { + // For contract creations, any storage node contained is a diff + var storageDiffs []StorageNode + err := sdb.buildStorageNodesEventual(val.Account.Root, watchedStorageKeys, intermediateStorageNodes, storageNodeAppender(&storageDiffs)) + if err != nil { + return fmt.Errorf("failed building eventual storage diffs for node %x\r\nerror: %v", val.Path, err) + } + diff.StorageNodes = storageDiffs + // emit codehash => code mappings for cod + codeHash := common.BytesToHash(val.Account.CodeHash) + code, err := sdb.stateCache.ContractCode(common.Hash{}, codeHash) + if err != nil { + return fmt.Errorf("failed to retrieve code for codehash %s\r\n error: %v", codeHash.String(), err) + } + if err := codeOutput(CodeAndCodeHash{ + Hash: codeHash, + Code: code, + }); err != nil { + return err + } + } + if err := output(diff); err != nil { + return err + } + } + + return nil +} + +// buildStorageNodesEventual builds the storage diff node objects for a created account +// i.e. it returns all the storage nodes at this state, since there is no previous state +func (sdb *builder) buildStorageNodesEventual(sr common.Hash, watchedStorageKeys []common.Hash, intermediateNodes bool, output StorageNodeSink) error { + if bytes.Equal(sr.Bytes(), emptyContractRoot.Bytes()) { + return nil + } + log.Debug("Storage Root For Eventual Diff", "root", sr.Hex()) + sTrie, err := sdb.stateCache.OpenTrie(sr) + if err != nil { + log.Info("error in build storage diff eventual", "error", err) + return err + } + it := sTrie.NodeIterator(make([]byte, 0)) + err = sdb.buildStorageNodesFromTrie(it, watchedStorageKeys, intermediateNodes, output) + if err != nil { + return err + } + return nil +} + +// buildStorageNodesFromTrie returns all the storage diff node objects in the provided node interator +// if any storage keys are provided it will only return those leaf nodes +// including intermediate nodes can be turned on or off +func (sdb *builder) buildStorageNodesFromTrie(it trie.NodeIterator, watchedStorageKeys []common.Hash, intermediateNodes bool, output StorageNodeSink) error { + for it.Next(true) { + // skip value nodes + if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) { + continue + } + node, nodeElements, err := resolveNode(it, sdb.stateCache.TrieDB()) + if err != nil { + return err + } + switch node.NodeType { + case Leaf: + partialPath := trie.CompactToHex(nodeElements[0].([]byte)) + valueNodePath := append(node.Path, partialPath...) + encodedPath := trie.HexToCompact(valueNodePath) + leafKey := encodedPath[1:] + if isWatchedStorageKey(watchedStorageKeys, leafKey) { + if err := output(StorageNode{ + NodeType: node.NodeType, + Path: node.Path, + NodeValue: node.NodeValue, + LeafKey: leafKey, + }); err != nil { + return err + } + } + case Extension, Branch: + if intermediateNodes { + if err := output(StorageNode{ + NodeType: node.NodeType, + Path: node.Path, + NodeValue: node.NodeValue, + }); err != nil { + return err + } + } + default: + return fmt.Errorf("unexpected node type %s", node.NodeType) + } + } + return it.Error() +} + +// buildStorageNodesIncremental builds the storage diff node objects for all nodes that exist in a different state at B than A +func (sdb *builder) buildStorageNodesIncremental(oldSR common.Hash, newSR common.Hash, watchedStorageKeys []common.Hash, intermediateNodes bool, output StorageNodeSink) error { + if bytes.Equal(newSR.Bytes(), oldSR.Bytes()) { + return nil + } + log.Debug("Storage Roots for Incremental Diff", "old", oldSR.Hex(), "new", newSR.Hex()) + oldTrie, err := sdb.stateCache.OpenTrie(oldSR) + if err != nil { + return err + } + newTrie, err := sdb.stateCache.OpenTrie(newSR) + if err != nil { + return err + } + + diffPathsAtB, err := sdb.createdAndUpdatedStorage( + oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), + watchedStorageKeys, intermediateNodes, output) + if err != nil { + return err + } + err = sdb.deletedOrUpdatedStorage(oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), + diffPathsAtB, watchedStorageKeys, intermediateNodes, output) + if err != nil { + return err + } + return nil +} + +func (sdb *builder) createdAndUpdatedStorage(a, b trie.NodeIterator, watchedKeys []common.Hash, intermediateNodes bool, output StorageNodeSink) (map[string]bool, error) { + diffPathsAtB := make(map[string]bool) + it, _ := trie.NewDifferenceIterator(a, b) + for it.Next(true) { + // skip value nodes + if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) { + continue + } + node, nodeElements, err := resolveNode(it, sdb.stateCache.TrieDB()) + if err != nil { + return nil, err + } + switch node.NodeType { + case Leaf: + partialPath := trie.CompactToHex(nodeElements[0].([]byte)) + valueNodePath := append(node.Path, partialPath...) + encodedPath := trie.HexToCompact(valueNodePath) + leafKey := encodedPath[1:] + if isWatchedStorageKey(watchedKeys, leafKey) { + if err := output(StorageNode{ + NodeType: node.NodeType, + Path: node.Path, + NodeValue: node.NodeValue, + LeafKey: leafKey, + }); err != nil { + return nil, err + } + } + case Extension, Branch: + if intermediateNodes { + if err := output(StorageNode{ + NodeType: node.NodeType, + Path: node.Path, + NodeValue: node.NodeValue, + }); err != nil { + return nil, err + } + } + default: + return nil, fmt.Errorf("unexpected node type %s", node.NodeType) + } + diffPathsAtB[common.Bytes2Hex(node.Path)] = true + } + return diffPathsAtB, it.Error() +} + +func (sdb *builder) deletedOrUpdatedStorage(a, b trie.NodeIterator, diffPathsAtB map[string]bool, watchedKeys []common.Hash, intermediateNodes bool, output StorageNodeSink) error { + it, _ := trie.NewDifferenceIterator(b, a) + for it.Next(true) { + // skip value nodes + if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) { + continue + } + node, nodeElements, err := resolveNode(it, sdb.stateCache.TrieDB()) + if err != nil { + return err + } + // if this node path showed up in diffPathsAtB + // that means this node was updated at B and we already have the updated diff for it + // otherwise that means this node was deleted in B and we need to add a "removed" diff to represent that event + if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; ok { + continue + } + switch node.NodeType { + case Leaf: + partialPath := trie.CompactToHex(nodeElements[0].([]byte)) + valueNodePath := append(node.Path, partialPath...) + encodedPath := trie.HexToCompact(valueNodePath) + leafKey := encodedPath[1:] + if isWatchedStorageKey(watchedKeys, leafKey) { + if err := output(StorageNode{ + NodeType: Removed, + Path: node.Path, + NodeValue: []byte{}, + }); err != nil { + return err + } + } + case Extension, Branch: + if intermediateNodes { + if err := output(StorageNode{ + NodeType: Removed, + Path: node.Path, + NodeValue: []byte{}, + }); err != nil { + return err + } + } + default: + return fmt.Errorf("unexpected node type %s", node.NodeType) + } + } + return it.Error() +} + +// isWatchedAddress is used to check if a state account corresponds to one of the addresses the builder is configured to watch +func isWatchedAddress(watchedAddresses []common.Address, stateLeafKey []byte) bool { + // If we aren't watching any specific addresses, we are watching everything + if len(watchedAddresses) == 0 { + return true + } + for _, addr := range watchedAddresses { + addrHashKey := crypto.Keccak256(addr.Bytes()) + if bytes.Equal(addrHashKey, stateLeafKey) { + return true + } + } + return false +} + +// isWatchedStorageKey is used to check if a storage leaf corresponds to one of the storage slots the builder is configured to watch +func isWatchedStorageKey(watchedKeys []common.Hash, storageLeafKey []byte) bool { + // If we aren't watching any specific addresses, we are watching everything + if len(watchedKeys) == 0 { + return true + } + for _, hashKey := range watchedKeys { + if bytes.Equal(hashKey.Bytes(), storageLeafKey) { + return true + } + } + return false +} diff --git a/statediff/builder_test.go b/statediff/builder_test.go new file mode 100644 index 000000000..8ad0d9822 --- /dev/null +++ b/statediff/builder_test.go @@ -0,0 +1,2300 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package statediff_test + +import ( + "bytes" + "math/big" + "sort" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/statediff" + "github.com/ethereum/go-ethereum/statediff/testhelpers" + sdtypes "github.com/ethereum/go-ethereum/statediff/types" +) + +// TODO: add test that filters on address +var ( + contractLeafKey []byte + emptyDiffs = make([]sdtypes.StateNode, 0) + emptyStorage = make([]sdtypes.StorageNode, 0) + block0, block1, block2, block3, block4, block5, block6 *types.Block + builder statediff.Builder + miningReward = int64(2000000000000000000) + minerAddress = common.HexToAddress("0x0") + minerLeafKey = testhelpers.AddressToLeafKey(minerAddress) + + balanceChange10000 = int64(10000) + balanceChange1000 = int64(1000) + block1BankBalance = int64(99990000) + block1Account1Balance = int64(10000) + block2Account2Balance = int64(1000) + + slot0 = common.HexToHash("0") + slot1 = common.HexToHash("1") + slot2 = common.HexToHash("2") + slot3 = common.HexToHash("3") + + slot0StorageKey = crypto.Keccak256Hash(slot0[:]) + slot1StorageKey = crypto.Keccak256Hash(slot1[:]) + slot2StorageKey = crypto.Keccak256Hash(slot2[:]) + slot3StorageKey = crypto.Keccak256Hash(slot3[:]) + + slot0StorageValue = common.Hex2Bytes("94703c4b2bd70c169f5717101caee543299fc946c7") // prefixed AccountAddr1 + slot1StorageValue = common.Hex2Bytes("01") + slot2StorageValue = common.Hex2Bytes("09") + slot3StorageValue = common.Hex2Bytes("03") + + slot0StorageLeafNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("390decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563"), + slot0StorageValue, + }) + slot1StorageLeafNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("310e2d527612073b26eecdfd717e6a320cf44b4afac2b0732d9fcbe2b7fa0cf6"), + slot1StorageValue, + }) + slot2StorageLeafNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("305787fa12a823e0f2b7631cc41b3ba8828b3321ca811111fa75cd3aa3bb5ace"), + slot2StorageValue, + }) + slot3StorageLeafNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("32575a0e9e593c00f959f8c92f12db2869c3395a3b0502d05e2516446f71f85b"), + slot3StorageValue, + }) + slot0StorageLeafRootNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("20290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563"), + slot0StorageValue, + }) + + contractAccountAtBlock2, _ = rlp.EncodeToBytes(state.Account{ + Nonce: 1, + Balance: big.NewInt(0), + CodeHash: common.HexToHash("0xaaea5efba4fd7b45d7ec03918ac5d8b31aa93b48986af0e6b591f0f087c80127").Bytes(), + Root: crypto.Keccak256Hash(block2StorageBranchRootNode), + }) + contractAccountAtBlock2LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("3114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45"), + contractAccountAtBlock2, + }) + contractAccountAtBlock3, _ = rlp.EncodeToBytes(state.Account{ + Nonce: 1, + Balance: big.NewInt(0), + CodeHash: common.HexToHash("0xaaea5efba4fd7b45d7ec03918ac5d8b31aa93b48986af0e6b591f0f087c80127").Bytes(), + Root: crypto.Keccak256Hash(block3StorageBranchRootNode), + }) + contractAccountAtBlock3LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("3114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45"), + contractAccountAtBlock3, + }) + contractAccountAtBlock4, _ = rlp.EncodeToBytes(state.Account{ + Nonce: 1, + Balance: big.NewInt(0), + CodeHash: common.HexToHash("0xaaea5efba4fd7b45d7ec03918ac5d8b31aa93b48986af0e6b591f0f087c80127").Bytes(), + Root: crypto.Keccak256Hash(block4StorageBranchRootNode), + }) + contractAccountAtBlock4LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("3114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45"), + contractAccountAtBlock4, + }) + contractAccountAtBlock5, _ = rlp.EncodeToBytes(state.Account{ + Nonce: 1, + Balance: big.NewInt(0), + CodeHash: common.HexToHash("0xaaea5efba4fd7b45d7ec03918ac5d8b31aa93b48986af0e6b591f0f087c80127").Bytes(), + Root: crypto.Keccak256Hash(slot0StorageLeafRootNode), + }) + contractAccountAtBlock5LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("3114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45"), + contractAccountAtBlock5, + }) + + minerAccountAtBlock1, _ = rlp.EncodeToBytes(state.Account{ + Nonce: 0, + Balance: big.NewInt(miningReward), + CodeHash: testhelpers.NullCodeHash.Bytes(), + Root: testhelpers.EmptyContractRoot, + }) + minerAccountAtBlock1LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("3380c7b7ae81a58eb98d9c78de4a1fd7fd9535fc953ed2be602daaa41767312a"), + minerAccountAtBlock1, + }) + minerAccountAtBlock2, _ = rlp.EncodeToBytes(state.Account{ + Nonce: 0, + Balance: big.NewInt(miningReward + miningReward), + CodeHash: testhelpers.NullCodeHash.Bytes(), + Root: testhelpers.EmptyContractRoot, + }) + minerAccountAtBlock2LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("3380c7b7ae81a58eb98d9c78de4a1fd7fd9535fc953ed2be602daaa41767312a"), + minerAccountAtBlock2, + }) + + account1AtBlock1, _ = rlp.EncodeToBytes(state.Account{ + Nonce: 0, + Balance: big.NewInt(balanceChange10000), + CodeHash: testhelpers.NullCodeHash.Bytes(), + Root: testhelpers.EmptyContractRoot, + }) + account1AtBlock1LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("3926db69aaced518e9b9f0f434a473e7174109c943548bb8f23be41ca76d9ad2"), + account1AtBlock1, + }) + account1AtBlock2, _ = rlp.EncodeToBytes(state.Account{ + Nonce: 2, + Balance: big.NewInt(block1Account1Balance - balanceChange1000 + balanceChange1000), + CodeHash: testhelpers.NullCodeHash.Bytes(), + Root: testhelpers.EmptyContractRoot, + }) + account1AtBlock2LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("3926db69aaced518e9b9f0f434a473e7174109c943548bb8f23be41ca76d9ad2"), + account1AtBlock2, + }) + account1AtBlock5, _ = rlp.EncodeToBytes(state.Account{ + Nonce: 2, + Balance: big.NewInt(block1Account1Balance - balanceChange1000 + balanceChange1000 + miningReward), + CodeHash: testhelpers.NullCodeHash.Bytes(), + Root: testhelpers.EmptyContractRoot, + }) + account1AtBlock5LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("3926db69aaced518e9b9f0f434a473e7174109c943548bb8f23be41ca76d9ad2"), + account1AtBlock5, + }) + account1AtBlock6, _ = rlp.EncodeToBytes(state.Account{ + Nonce: 3, + Balance: big.NewInt(block1Account1Balance - balanceChange1000 + balanceChange1000 + miningReward), + CodeHash: testhelpers.NullCodeHash.Bytes(), + Root: testhelpers.EmptyContractRoot, + }) + account1AtBlock6LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("3926db69aaced518e9b9f0f434a473e7174109c943548bb8f23be41ca76d9ad2"), + account1AtBlock6, + }) + + account2AtBlock2, _ = rlp.EncodeToBytes(state.Account{ + Nonce: 0, + Balance: big.NewInt(balanceChange1000), + CodeHash: testhelpers.NullCodeHash.Bytes(), + Root: testhelpers.EmptyContractRoot, + }) + account2AtBlock2LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("3957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45"), + account2AtBlock2, + }) + account2AtBlock3, _ = rlp.EncodeToBytes(state.Account{ + Nonce: 0, + Balance: big.NewInt(block2Account2Balance + miningReward), + CodeHash: testhelpers.NullCodeHash.Bytes(), + Root: testhelpers.EmptyContractRoot, + }) + account2AtBlock3LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("3957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45"), + account2AtBlock3, + }) + account2AtBlock4, _ = rlp.EncodeToBytes(state.Account{ + Nonce: 0, + Balance: big.NewInt(block2Account2Balance + miningReward*2), + CodeHash: testhelpers.NullCodeHash.Bytes(), + Root: testhelpers.EmptyContractRoot, + }) + account2AtBlock4LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("3957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45"), + account2AtBlock4, + }) + account2AtBlock6, _ = rlp.EncodeToBytes(state.Account{ + Nonce: 0, + Balance: big.NewInt(block2Account2Balance + miningReward*3), + CodeHash: testhelpers.NullCodeHash.Bytes(), + Root: testhelpers.EmptyContractRoot, + }) + account2AtBlock6LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("3957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45"), + account2AtBlock6, + }) + + bankAccountAtBlock0, _ = rlp.EncodeToBytes(state.Account{ + Nonce: 0, + Balance: big.NewInt(testhelpers.TestBankFunds.Int64()), + CodeHash: testhelpers.NullCodeHash.Bytes(), + Root: testhelpers.EmptyContractRoot, + }) + bankAccountAtBlock0LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("2000bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a"), + bankAccountAtBlock0, + }) + bankAccountAtBlock1, _ = rlp.EncodeToBytes(state.Account{ + Nonce: 1, + Balance: big.NewInt(testhelpers.TestBankFunds.Int64() - balanceChange10000), + CodeHash: testhelpers.NullCodeHash.Bytes(), + Root: testhelpers.EmptyContractRoot, + }) + bankAccountAtBlock1LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("30bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a"), + bankAccountAtBlock1, + }) + bankAccountAtBlock2, _ = rlp.EncodeToBytes(state.Account{ + Nonce: 2, + Balance: big.NewInt(block1BankBalance - balanceChange1000), + CodeHash: testhelpers.NullCodeHash.Bytes(), + Root: testhelpers.EmptyContractRoot, + }) + bankAccountAtBlock2LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("30bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a"), + bankAccountAtBlock2, + }) + bankAccountAtBlock3, _ = rlp.EncodeToBytes(state.Account{ + Nonce: 3, + Balance: big.NewInt(99989000), + CodeHash: testhelpers.NullCodeHash.Bytes(), + Root: testhelpers.EmptyContractRoot, + }) + bankAccountAtBlock3LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("30bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a"), + bankAccountAtBlock3, + }) + bankAccountAtBlock4, _ = rlp.EncodeToBytes(state.Account{ + Nonce: 6, + Balance: big.NewInt(99989000), + CodeHash: testhelpers.NullCodeHash.Bytes(), + Root: testhelpers.EmptyContractRoot, + }) + bankAccountAtBlock4LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("30bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a"), + bankAccountAtBlock4, + }) + bankAccountAtBlock5, _ = rlp.EncodeToBytes(state.Account{ + Nonce: 7, + Balance: big.NewInt(99989000), + CodeHash: testhelpers.NullCodeHash.Bytes(), + Root: testhelpers.EmptyContractRoot, + }) + bankAccountAtBlock5LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("30bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a"), + bankAccountAtBlock5, + }) + + block1BranchRootNode, _ = rlp.EncodeToBytes([]interface{}{ + crypto.Keccak256(bankAccountAtBlock1LeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + crypto.Keccak256(minerAccountAtBlock1LeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + crypto.Keccak256(account1AtBlock1LeafNode), + []byte{}, + []byte{}, + }) + block2BranchRootNode, _ = rlp.EncodeToBytes([]interface{}{ + crypto.Keccak256(bankAccountAtBlock2LeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + crypto.Keccak256(minerAccountAtBlock2LeafNode), + crypto.Keccak256(contractAccountAtBlock2LeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + crypto.Keccak256(account2AtBlock2LeafNode), + []byte{}, + crypto.Keccak256(account1AtBlock2LeafNode), + []byte{}, + []byte{}, + }) + block3BranchRootNode, _ = rlp.EncodeToBytes([]interface{}{ + crypto.Keccak256(bankAccountAtBlock3LeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + crypto.Keccak256(minerAccountAtBlock2LeafNode), + crypto.Keccak256(contractAccountAtBlock3LeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + crypto.Keccak256(account2AtBlock3LeafNode), + []byte{}, + crypto.Keccak256(account1AtBlock2LeafNode), + []byte{}, + []byte{}, + }) + block4BranchRootNode, _ = rlp.EncodeToBytes([]interface{}{ + crypto.Keccak256(bankAccountAtBlock4LeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + crypto.Keccak256(minerAccountAtBlock2LeafNode), + crypto.Keccak256(contractAccountAtBlock4LeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + crypto.Keccak256(account2AtBlock4LeafNode), + []byte{}, + crypto.Keccak256(account1AtBlock2LeafNode), + []byte{}, + []byte{}, + }) + block5BranchRootNode, _ = rlp.EncodeToBytes([]interface{}{ + crypto.Keccak256(bankAccountAtBlock5LeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + crypto.Keccak256(minerAccountAtBlock2LeafNode), + crypto.Keccak256(contractAccountAtBlock5LeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + crypto.Keccak256(account2AtBlock4LeafNode), + []byte{}, + crypto.Keccak256(account1AtBlock5LeafNode), + []byte{}, + []byte{}, + }) + block6BranchRootNode, _ = rlp.EncodeToBytes([]interface{}{ + crypto.Keccak256(bankAccountAtBlock5LeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + crypto.Keccak256(minerAccountAtBlock2LeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + crypto.Keccak256(account2AtBlock6LeafNode), + []byte{}, + crypto.Keccak256(account1AtBlock6LeafNode), + []byte{}, + []byte{}, + }) + + block2StorageBranchRootNode, _ = rlp.EncodeToBytes([]interface{}{ + []byte{}, + []byte{}, + crypto.Keccak256(slot0StorageLeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + crypto.Keccak256(slot1StorageLeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + }) + block3StorageBranchRootNode, _ = rlp.EncodeToBytes([]interface{}{ + []byte{}, + []byte{}, + crypto.Keccak256(slot0StorageLeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + crypto.Keccak256(slot1StorageLeafNode), + crypto.Keccak256(slot3StorageLeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + }) + block4StorageBranchRootNode, _ = rlp.EncodeToBytes([]interface{}{ + []byte{}, + []byte{}, + crypto.Keccak256(slot0StorageLeafNode), + []byte{}, + crypto.Keccak256(slot2StorageLeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + }) +) + +func TestBuilder(t *testing.T) { + blocks, chain := testhelpers.MakeChain(3, testhelpers.Genesis, testhelpers.TestChainGen) + contractLeafKey = testhelpers.AddressToLeafKey(testhelpers.ContractAddr) + defer chain.Stop() + block0 = testhelpers.Genesis + block1 = blocks[0] + block2 = blocks[1] + block3 = blocks[2] + params := statediff.Params{} + builder = statediff.NewBuilder(chain.StateCache()) + + var tests = []struct { + name string + startingArguments statediff.Args + expected *statediff.StateObject + }{ + { + "testEmptyDiff", + statediff.Args{ + OldStateRoot: block0.Root(), + NewStateRoot: block0.Root(), + BlockNumber: block0.Number(), + BlockHash: block0.Hash(), + }, + &statediff.StateObject{ + BlockNumber: block0.Number(), + BlockHash: block0.Hash(), + Nodes: emptyDiffs, + }, + }, + { + "testBlock0", + //10000 transferred from testBankAddress to account1Addr + statediff.Args{ + OldStateRoot: testhelpers.NullHash, + NewStateRoot: block0.Root(), + BlockNumber: block0.Number(), + BlockHash: block0.Hash(), + }, + &statediff.StateObject{ + BlockNumber: block0.Number(), + BlockHash: block0.Hash(), + Nodes: []sdtypes.StateNode{ + { + Path: []byte{}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.BankLeafKey, + NodeValue: bankAccountAtBlock0LeafNode, + StorageNodes: emptyStorage, + }, + }, + }, + }, + { + "testBlock1", + //10000 transferred from testBankAddress to account1Addr + statediff.Args{ + OldStateRoot: block0.Root(), + NewStateRoot: block1.Root(), + BlockNumber: block1.Number(), + BlockHash: block1.Hash(), + }, + &statediff.StateObject{ + BlockNumber: block1.Number(), + BlockHash: block1.Hash(), + Nodes: []sdtypes.StateNode{ + { + Path: []byte{'\x00'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.BankLeafKey, + NodeValue: bankAccountAtBlock1LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x05'}, + NodeType: sdtypes.Leaf, + LeafKey: minerLeafKey, + NodeValue: minerAccountAtBlock1LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x0e'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.Account1LeafKey, + NodeValue: account1AtBlock1LeafNode, + StorageNodes: emptyStorage, + }, + }, + }, + }, + { + "testBlock2", + // 1000 transferred from testBankAddress to account1Addr + // 1000 transferred from account1Addr to account2Addr + // account1addr creates a new contract + statediff.Args{ + OldStateRoot: block1.Root(), + NewStateRoot: block2.Root(), + BlockNumber: block2.Number(), + BlockHash: block2.Hash(), + }, + &statediff.StateObject{ + BlockNumber: block2.Number(), + BlockHash: block2.Hash(), + Nodes: []sdtypes.StateNode{ + { + Path: []byte{'\x00'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.BankLeafKey, + NodeValue: bankAccountAtBlock2LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x05'}, + NodeType: sdtypes.Leaf, + LeafKey: minerLeafKey, + NodeValue: minerAccountAtBlock2LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x0e'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.Account1LeafKey, + NodeValue: account1AtBlock2LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x06'}, + NodeType: sdtypes.Leaf, + LeafKey: contractLeafKey, + NodeValue: contractAccountAtBlock2LeafNode, + StorageNodes: []sdtypes.StorageNode{ + { + Path: []byte{'\x02'}, + NodeType: sdtypes.Leaf, + LeafKey: slot0StorageKey.Bytes(), + NodeValue: slot0StorageLeafNode, + }, + { + Path: []byte{'\x0b'}, + NodeType: sdtypes.Leaf, + LeafKey: slot1StorageKey.Bytes(), + NodeValue: slot1StorageLeafNode, + }, + }, + }, + { + Path: []byte{'\x0c'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.Account2LeafKey, + NodeValue: account2AtBlock2LeafNode, + StorageNodes: emptyStorage, + }, + }, + CodeAndCodeHashes: []sdtypes.CodeAndCodeHash{ + { + Hash: testhelpers.CodeHash, + Code: testhelpers.ByteCodeAfterDeployment, + }, + }, + }, + }, + { + "testBlock3", + //the contract's storage is changed + //and the block is mined by account 2 + statediff.Args{ + OldStateRoot: block2.Root(), + NewStateRoot: block3.Root(), + BlockNumber: block3.Number(), + BlockHash: block3.Hash(), + }, + &statediff.StateObject{ + BlockNumber: block3.Number(), + BlockHash: block3.Hash(), + Nodes: []sdtypes.StateNode{ + { + Path: []byte{'\x00'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.BankLeafKey, + NodeValue: bankAccountAtBlock3LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x06'}, + NodeType: sdtypes.Leaf, + LeafKey: contractLeafKey, + NodeValue: contractAccountAtBlock3LeafNode, + StorageNodes: []sdtypes.StorageNode{ + { + Path: []byte{'\x0c'}, + NodeType: sdtypes.Leaf, + LeafKey: slot3StorageKey.Bytes(), + NodeValue: slot3StorageLeafNode, + }, + }, + }, + { + Path: []byte{'\x0c'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.Account2LeafKey, + NodeValue: account2AtBlock3LeafNode, + StorageNodes: emptyStorage, + }, + }, + }, + }, + } + + for _, test := range tests { + diff, err := builder.BuildStateDiffObject(test.startingArguments, params) + if err != nil { + t.Error(err) + } + receivedStateDiffRlp, err := rlp.EncodeToBytes(diff) + if err != nil { + t.Error(err) + } + expectedStateDiffRlp, err := rlp.EncodeToBytes(test.expected) + if err != nil { + t.Error(err) + } + sort.Slice(receivedStateDiffRlp, func(i, j int) bool { return receivedStateDiffRlp[i] < receivedStateDiffRlp[j] }) + sort.Slice(expectedStateDiffRlp, func(i, j int) bool { return expectedStateDiffRlp[i] < expectedStateDiffRlp[j] }) + if !bytes.Equal(receivedStateDiffRlp, expectedStateDiffRlp) { + t.Logf("Test failed: %s", test.name) + t.Errorf("actual state diff: %+v\nexpected state diff: %+v", diff, test.expected) + } + } +} + +func TestBuilderWithIntermediateNodes(t *testing.T) { + blocks, chain := testhelpers.MakeChain(3, testhelpers.Genesis, testhelpers.TestChainGen) + contractLeafKey = testhelpers.AddressToLeafKey(testhelpers.ContractAddr) + defer chain.Stop() + block0 = testhelpers.Genesis + block1 = blocks[0] + block2 = blocks[1] + block3 = blocks[2] + blocks = append([]*types.Block{block0}, blocks...) + params := statediff.Params{ + IntermediateStateNodes: true, + IntermediateStorageNodes: true, + } + builder = statediff.NewBuilder(chain.StateCache()) + + var tests = []struct { + name string + startingArguments statediff.Args + expected *statediff.StateObject + }{ + { + "testEmptyDiff", + statediff.Args{ + OldStateRoot: block0.Root(), + NewStateRoot: block0.Root(), + BlockNumber: block0.Number(), + BlockHash: block0.Hash(), + }, + &statediff.StateObject{ + BlockNumber: block0.Number(), + BlockHash: block0.Hash(), + Nodes: emptyDiffs, + }, + }, + { + "testBlock0", + //10000 transferred from testBankAddress to account1Addr + statediff.Args{ + OldStateRoot: testhelpers.NullHash, + NewStateRoot: block0.Root(), + BlockNumber: block0.Number(), + BlockHash: block0.Hash(), + }, + &statediff.StateObject{ + BlockNumber: block0.Number(), + BlockHash: block0.Hash(), + Nodes: []sdtypes.StateNode{ + { + Path: []byte{}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.BankLeafKey, + NodeValue: bankAccountAtBlock0LeafNode, + StorageNodes: emptyStorage, + }, + }, + }, + }, + { + "testBlock1", + //10000 transferred from testBankAddress to account1Addr + statediff.Args{ + OldStateRoot: block0.Root(), + NewStateRoot: block1.Root(), + BlockNumber: block1.Number(), + BlockHash: block1.Hash(), + }, + &statediff.StateObject{ + BlockNumber: block1.Number(), + BlockHash: block1.Hash(), + Nodes: []sdtypes.StateNode{ + { + Path: []byte{}, + NodeType: sdtypes.Branch, + NodeValue: block1BranchRootNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x00'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.BankLeafKey, + NodeValue: bankAccountAtBlock1LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x05'}, + NodeType: sdtypes.Leaf, + LeafKey: minerLeafKey, + NodeValue: minerAccountAtBlock1LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x0e'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.Account1LeafKey, + NodeValue: account1AtBlock1LeafNode, + StorageNodes: emptyStorage, + }, + }, + }, + }, + { + "testBlock2", + // 1000 transferred from testBankAddress to account1Addr + // 1000 transferred from account1Addr to account2Addr + // account1addr creates a new contract + statediff.Args{ + OldStateRoot: block1.Root(), + NewStateRoot: block2.Root(), + BlockNumber: block2.Number(), + BlockHash: block2.Hash(), + }, + &statediff.StateObject{ + BlockNumber: block2.Number(), + BlockHash: block2.Hash(), + Nodes: []sdtypes.StateNode{ + { + Path: []byte{}, + NodeType: sdtypes.Branch, + NodeValue: block2BranchRootNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x00'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.BankLeafKey, + NodeValue: bankAccountAtBlock2LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x05'}, + NodeType: sdtypes.Leaf, + LeafKey: minerLeafKey, + NodeValue: minerAccountAtBlock2LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x0e'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.Account1LeafKey, + NodeValue: account1AtBlock2LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x06'}, + NodeType: sdtypes.Leaf, + LeafKey: contractLeafKey, + NodeValue: contractAccountAtBlock2LeafNode, + StorageNodes: []sdtypes.StorageNode{ + { + Path: []byte{}, + NodeType: sdtypes.Branch, + NodeValue: block2StorageBranchRootNode, + }, + { + Path: []byte{'\x02'}, + NodeType: sdtypes.Leaf, + LeafKey: slot0StorageKey.Bytes(), + NodeValue: slot0StorageLeafNode, + }, + { + Path: []byte{'\x0b'}, + NodeType: sdtypes.Leaf, + LeafKey: slot1StorageKey.Bytes(), + NodeValue: slot1StorageLeafNode, + }, + }, + }, + { + Path: []byte{'\x0c'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.Account2LeafKey, + NodeValue: account2AtBlock2LeafNode, + StorageNodes: emptyStorage, + }, + }, + CodeAndCodeHashes: []sdtypes.CodeAndCodeHash{ + { + Hash: testhelpers.CodeHash, + Code: testhelpers.ByteCodeAfterDeployment, + }, + }, + }, + }, + { + "testBlock3", + //the contract's storage is changed + //and the block is mined by account 2 + statediff.Args{ + OldStateRoot: block2.Root(), + NewStateRoot: block3.Root(), + BlockNumber: block3.Number(), + BlockHash: block3.Hash(), + }, + &statediff.StateObject{ + BlockNumber: block3.Number(), + BlockHash: block3.Hash(), + Nodes: []sdtypes.StateNode{ + { + Path: []byte{}, + NodeType: sdtypes.Branch, + NodeValue: block3BranchRootNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x00'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.BankLeafKey, + NodeValue: bankAccountAtBlock3LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x06'}, + NodeType: sdtypes.Leaf, + LeafKey: contractLeafKey, + NodeValue: contractAccountAtBlock3LeafNode, + StorageNodes: []sdtypes.StorageNode{ + { + Path: []byte{}, + NodeType: sdtypes.Branch, + NodeValue: block3StorageBranchRootNode, + }, + { + Path: []byte{'\x0c'}, + NodeType: sdtypes.Leaf, + LeafKey: slot3StorageKey.Bytes(), + NodeValue: slot3StorageLeafNode, + }, + }, + }, + { + Path: []byte{'\x0c'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.Account2LeafKey, + NodeValue: account2AtBlock3LeafNode, + StorageNodes: emptyStorage, + }, + }, + }, + }, + } + + for i, test := range tests { + diff, err := builder.BuildStateDiffObject(test.startingArguments, params) + if err != nil { + t.Error(err) + } + receivedStateDiffRlp, err := rlp.EncodeToBytes(diff) + if err != nil { + t.Error(err) + } + expectedStateDiffRlp, err := rlp.EncodeToBytes(test.expected) + if err != nil { + t.Error(err) + } + sort.Slice(receivedStateDiffRlp, func(i, j int) bool { return receivedStateDiffRlp[i] < receivedStateDiffRlp[j] }) + sort.Slice(expectedStateDiffRlp, func(i, j int) bool { return expectedStateDiffRlp[i] < expectedStateDiffRlp[j] }) + if !bytes.Equal(receivedStateDiffRlp, expectedStateDiffRlp) { + t.Logf("Test failed: %s", test.name) + t.Errorf("actual state diff: %+v\r\n\r\n\r\nexpected state diff: %+v", diff, test.expected) + } + // Let's also confirm that our root state nodes form the state root hash in the headers + if i > 0 { + block := blocks[i-1] + expectedStateRoot := block.Root() + for _, node := range test.expected.Nodes { + if bytes.Equal(node.Path, []byte{}) { + stateRoot := crypto.Keccak256Hash(node.NodeValue) + if !bytes.Equal(expectedStateRoot.Bytes(), stateRoot.Bytes()) { + t.Logf("Test failed: %s", test.name) + t.Errorf("actual stateroot: %x\r\nexpected stateroot: %x", stateRoot.Bytes(), expectedStateRoot.Bytes()) + } + } + } + } + } +} + +func TestBuilderWithWatchedAddressList(t *testing.T) { + blocks, chain := testhelpers.MakeChain(3, testhelpers.Genesis, testhelpers.TestChainGen) + contractLeafKey = testhelpers.AddressToLeafKey(testhelpers.ContractAddr) + defer chain.Stop() + block0 = testhelpers.Genesis + block1 = blocks[0] + block2 = blocks[1] + block3 = blocks[2] + params := statediff.Params{ + WatchedAddresses: []common.Address{testhelpers.Account1Addr, testhelpers.ContractAddr}, + } + builder = statediff.NewBuilder(chain.StateCache()) + + var tests = []struct { + name string + startingArguments statediff.Args + expected *statediff.StateObject + }{ + { + "testEmptyDiff", + statediff.Args{ + OldStateRoot: block0.Root(), + NewStateRoot: block0.Root(), + BlockNumber: block0.Number(), + BlockHash: block0.Hash(), + }, + &statediff.StateObject{ + BlockNumber: block0.Number(), + BlockHash: block0.Hash(), + Nodes: emptyDiffs, + }, + }, + { + "testBlock0", + //10000 transferred from testBankAddress to account1Addr + statediff.Args{ + OldStateRoot: testhelpers.NullHash, + NewStateRoot: block0.Root(), + BlockNumber: block0.Number(), + BlockHash: block0.Hash(), + }, + &statediff.StateObject{ + BlockNumber: block0.Number(), + BlockHash: block0.Hash(), + Nodes: emptyDiffs, + }, + }, + { + "testBlock1", + //10000 transferred from testBankAddress to account1Addr + statediff.Args{ + OldStateRoot: block0.Root(), + NewStateRoot: block1.Root(), + BlockNumber: block1.Number(), + BlockHash: block1.Hash(), + }, + &statediff.StateObject{ + BlockNumber: block1.Number(), + BlockHash: block1.Hash(), + Nodes: []sdtypes.StateNode{ + { + Path: []byte{'\x0e'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.Account1LeafKey, + NodeValue: account1AtBlock1LeafNode, + StorageNodes: emptyStorage, + }, + }, + }, + }, + { + "testBlock2", + //1000 transferred from testBankAddress to account1Addr + //1000 transferred from account1Addr to account2Addr + statediff.Args{ + OldStateRoot: block1.Root(), + NewStateRoot: block2.Root(), + BlockNumber: block2.Number(), + BlockHash: block2.Hash(), + }, + &statediff.StateObject{ + BlockNumber: block2.Number(), + BlockHash: block2.Hash(), + Nodes: []sdtypes.StateNode{ + { + Path: []byte{'\x06'}, + NodeType: sdtypes.Leaf, + LeafKey: contractLeafKey, + NodeValue: contractAccountAtBlock2LeafNode, + StorageNodes: []sdtypes.StorageNode{ + { + Path: []byte{'\x02'}, + NodeType: sdtypes.Leaf, + LeafKey: slot0StorageKey.Bytes(), + NodeValue: slot0StorageLeafNode, + }, + { + Path: []byte{'\x0b'}, + NodeType: sdtypes.Leaf, + LeafKey: slot1StorageKey.Bytes(), + NodeValue: slot1StorageLeafNode, + }, + }, + }, + { + Path: []byte{'\x0e'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.Account1LeafKey, + NodeValue: account1AtBlock2LeafNode, + StorageNodes: emptyStorage, + }, + }, + CodeAndCodeHashes: []sdtypes.CodeAndCodeHash{ + { + Hash: testhelpers.CodeHash, + Code: testhelpers.ByteCodeAfterDeployment, + }, + }, + }, + }, + { + "testBlock3", + //the contract's storage is changed + //and the block is mined by account 2 + statediff.Args{ + OldStateRoot: block2.Root(), + NewStateRoot: block3.Root(), + BlockNumber: block3.Number(), + BlockHash: block3.Hash(), + }, + &statediff.StateObject{ + BlockNumber: block3.Number(), + BlockHash: block3.Hash(), + Nodes: []sdtypes.StateNode{ + { + Path: []byte{'\x06'}, + NodeType: sdtypes.Leaf, + LeafKey: contractLeafKey, + NodeValue: contractAccountAtBlock3LeafNode, + StorageNodes: []sdtypes.StorageNode{ + { + Path: []byte{'\x0c'}, + NodeType: sdtypes.Leaf, + LeafKey: slot3StorageKey.Bytes(), + NodeValue: slot3StorageLeafNode, + }, + }, + }, + }, + }, + }, + } + + for _, test := range tests { + diff, err := builder.BuildStateDiffObject(test.startingArguments, params) + if err != nil { + t.Error(err) + } + receivedStateDiffRlp, err := rlp.EncodeToBytes(diff) + if err != nil { + t.Error(err) + } + expectedStateDiffRlp, err := rlp.EncodeToBytes(test.expected) + if err != nil { + t.Error(err) + } + sort.Slice(receivedStateDiffRlp, func(i, j int) bool { return receivedStateDiffRlp[i] < receivedStateDiffRlp[j] }) + sort.Slice(expectedStateDiffRlp, func(i, j int) bool { return expectedStateDiffRlp[i] < expectedStateDiffRlp[j] }) + if !bytes.Equal(receivedStateDiffRlp, expectedStateDiffRlp) { + t.Logf("Test failed: %s", test.name) + t.Errorf("actual state diff: %+v\nexpected state diff: %+v", diff, test.expected) + } + } +} + +func TestBuilderWithWatchedAddressAndStorageKeyList(t *testing.T) { + blocks, chain := testhelpers.MakeChain(3, testhelpers.Genesis, testhelpers.TestChainGen) + contractLeafKey = testhelpers.AddressToLeafKey(testhelpers.ContractAddr) + defer chain.Stop() + block0 = testhelpers.Genesis + block1 = blocks[0] + block2 = blocks[1] + block3 = blocks[2] + params := statediff.Params{ + WatchedAddresses: []common.Address{testhelpers.Account1Addr, testhelpers.ContractAddr}, + WatchedStorageSlots: []common.Hash{slot1StorageKey}, + } + builder = statediff.NewBuilder(chain.StateCache()) + + var tests = []struct { + name string + startingArguments statediff.Args + expected *statediff.StateObject + }{ + { + "testEmptyDiff", + statediff.Args{ + OldStateRoot: block0.Root(), + NewStateRoot: block0.Root(), + BlockNumber: block0.Number(), + BlockHash: block0.Hash(), + }, + &statediff.StateObject{ + BlockNumber: block0.Number(), + BlockHash: block0.Hash(), + Nodes: emptyDiffs, + }, + }, + { + "testBlock0", + //10000 transferred from testBankAddress to account1Addr + statediff.Args{ + OldStateRoot: testhelpers.NullHash, + NewStateRoot: block0.Root(), + BlockNumber: block0.Number(), + BlockHash: block0.Hash(), + }, + &statediff.StateObject{ + BlockNumber: block0.Number(), + BlockHash: block0.Hash(), + Nodes: emptyDiffs, + }, + }, + { + "testBlock1", + //10000 transferred from testBankAddress to account1Addr + statediff.Args{ + OldStateRoot: block0.Root(), + NewStateRoot: block1.Root(), + BlockNumber: block1.Number(), + BlockHash: block1.Hash(), + }, + &statediff.StateObject{ + BlockNumber: block1.Number(), + BlockHash: block1.Hash(), + Nodes: []sdtypes.StateNode{ + { + Path: []byte{'\x0e'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.Account1LeafKey, + NodeValue: account1AtBlock1LeafNode, + StorageNodes: emptyStorage, + }, + }, + }, + }, + { + "testBlock2", + //1000 transferred from testBankAddress to account1Addr + //1000 transferred from account1Addr to account2Addr + statediff.Args{ + OldStateRoot: block1.Root(), + NewStateRoot: block2.Root(), + BlockNumber: block2.Number(), + BlockHash: block2.Hash(), + }, + &statediff.StateObject{ + BlockNumber: block2.Number(), + BlockHash: block2.Hash(), + Nodes: []sdtypes.StateNode{ + { + Path: []byte{'\x06'}, + NodeType: sdtypes.Leaf, + LeafKey: contractLeafKey, + NodeValue: contractAccountAtBlock2LeafNode, + StorageNodes: []sdtypes.StorageNode{ + { + Path: []byte{'\x0b'}, + NodeType: sdtypes.Leaf, + LeafKey: slot1StorageKey.Bytes(), + NodeValue: slot1StorageLeafNode, + }, + }, + }, + { + Path: []byte{'\x0e'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.Account1LeafKey, + NodeValue: account1AtBlock2LeafNode, + StorageNodes: emptyStorage, + }, + }, + CodeAndCodeHashes: []sdtypes.CodeAndCodeHash{ + { + Hash: testhelpers.CodeHash, + Code: testhelpers.ByteCodeAfterDeployment, + }, + }, + }, + }, + { + "testBlock3", + //the contract's storage is changed + //and the block is mined by account 2 + statediff.Args{ + OldStateRoot: block2.Root(), + NewStateRoot: block3.Root(), + BlockNumber: block3.Number(), + BlockHash: block3.Hash(), + }, + &statediff.StateObject{ + BlockNumber: block3.Number(), + BlockHash: block3.Hash(), + Nodes: []sdtypes.StateNode{ + { + Path: []byte{'\x06'}, + NodeType: sdtypes.Leaf, + LeafKey: contractLeafKey, + NodeValue: contractAccountAtBlock3LeafNode, + StorageNodes: emptyStorage, + }, + }, + }, + }, + } + + for _, test := range tests { + diff, err := builder.BuildStateDiffObject(test.startingArguments, params) + if err != nil { + t.Error(err) + } + receivedStateDiffRlp, err := rlp.EncodeToBytes(diff) + if err != nil { + t.Error(err) + } + expectedStateDiffRlp, err := rlp.EncodeToBytes(test.expected) + if err != nil { + t.Error(err) + } + sort.Slice(receivedStateDiffRlp, func(i, j int) bool { return receivedStateDiffRlp[i] < receivedStateDiffRlp[j] }) + sort.Slice(expectedStateDiffRlp, func(i, j int) bool { return expectedStateDiffRlp[i] < expectedStateDiffRlp[j] }) + if !bytes.Equal(receivedStateDiffRlp, expectedStateDiffRlp) { + t.Logf("Test failed: %s", test.name) + t.Errorf("actual state diff: %+v\nexpected state diff: %+v", diff, test.expected) + } + } +} + +func TestBuilderWithRemovedAccountAndStorage(t *testing.T) { + blocks, chain := testhelpers.MakeChain(6, testhelpers.Genesis, testhelpers.TestChainGen) + contractLeafKey = testhelpers.AddressToLeafKey(testhelpers.ContractAddr) + defer chain.Stop() + block3 = blocks[2] + block4 = blocks[3] + block5 = blocks[4] + block6 = blocks[5] + params := statediff.Params{ + IntermediateStateNodes: true, + IntermediateStorageNodes: true, + } + builder = statediff.NewBuilder(chain.StateCache()) + + var tests = []struct { + name string + startingArguments statediff.Args + expected *statediff.StateObject + }{ + // blocks 0-3 are the same as in TestBuilderWithIntermediateNodes + { + "testBlock4", + statediff.Args{ + OldStateRoot: block3.Root(), + NewStateRoot: block4.Root(), + BlockNumber: block4.Number(), + BlockHash: block4.Hash(), + }, + &statediff.StateObject{ + BlockNumber: block4.Number(), + BlockHash: block4.Hash(), + Nodes: []sdtypes.StateNode{ + { + Path: []byte{}, + NodeType: sdtypes.Branch, + NodeValue: block4BranchRootNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x00'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.BankLeafKey, + NodeValue: bankAccountAtBlock4LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x06'}, + NodeType: sdtypes.Leaf, + LeafKey: contractLeafKey, + NodeValue: contractAccountAtBlock4LeafNode, + StorageNodes: []sdtypes.StorageNode{ + { + Path: []byte{}, + NodeType: sdtypes.Branch, + NodeValue: block4StorageBranchRootNode, + }, + { + Path: []byte{'\x04'}, + NodeType: sdtypes.Leaf, + LeafKey: slot2StorageKey.Bytes(), + NodeValue: slot2StorageLeafNode, + }, + { + Path: []byte{'\x0b'}, + NodeType: sdtypes.Removed, + NodeValue: []byte{}, + }, + { + Path: []byte{'\x0c'}, + NodeType: sdtypes.Removed, + NodeValue: []byte{}, + }, + }, + }, + { + Path: []byte{'\x0c'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.Account2LeafKey, + NodeValue: account2AtBlock4LeafNode, + StorageNodes: emptyStorage, + }, + }, + }, + }, + { + "testBlock5", + statediff.Args{ + OldStateRoot: block4.Root(), + NewStateRoot: block5.Root(), + BlockNumber: block5.Number(), + BlockHash: block5.Hash(), + }, + &statediff.StateObject{ + BlockNumber: block5.Number(), + BlockHash: block5.Hash(), + Nodes: []sdtypes.StateNode{ + { + Path: []byte{}, + NodeType: sdtypes.Branch, + NodeValue: block5BranchRootNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x00'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.BankLeafKey, + NodeValue: bankAccountAtBlock5LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x06'}, + NodeType: sdtypes.Leaf, + LeafKey: contractLeafKey, + NodeValue: contractAccountAtBlock5LeafNode, + StorageNodes: []sdtypes.StorageNode{ + { + Path: []byte{}, + NodeType: sdtypes.Leaf, + NodeValue: slot0StorageLeafRootNode, + LeafKey: slot0StorageKey.Bytes(), + }, + { + Path: []byte{'\x02'}, + NodeType: sdtypes.Removed, + NodeValue: []byte{}, + }, + { + Path: []byte{'\x04'}, + NodeType: sdtypes.Removed, + NodeValue: []byte{}, + }, + }, + }, + { + Path: []byte{'\x0e'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.Account1LeafKey, + NodeValue: account1AtBlock5LeafNode, + StorageNodes: emptyStorage, + }, + }, + }, + }, + { + "testBlock6", + statediff.Args{ + OldStateRoot: block5.Root(), + NewStateRoot: block6.Root(), + BlockNumber: block6.Number(), + BlockHash: block6.Hash(), + }, + &statediff.StateObject{ + BlockNumber: block6.Number(), + BlockHash: block6.Hash(), + Nodes: []sdtypes.StateNode{ + { + Path: []byte{}, + NodeType: sdtypes.Branch, + NodeValue: block6BranchRootNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x06'}, + NodeType: sdtypes.Removed, + NodeValue: []byte{}, + }, + { + Path: []byte{'\x0c'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.Account2LeafKey, + NodeValue: account2AtBlock6LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x0e'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.Account1LeafKey, + NodeValue: account1AtBlock6LeafNode, + StorageNodes: emptyStorage, + }, + }, + }, + }, + } + + for _, test := range tests { + diff, err := builder.BuildStateDiffObject(test.startingArguments, params) + if err != nil { + t.Error(err) + } + receivedStateDiffRlp, err := rlp.EncodeToBytes(diff) + if err != nil { + t.Error(err) + } + expectedStateDiffRlp, err := rlp.EncodeToBytes(test.expected) + if err != nil { + t.Error(err) + } + sort.Slice(receivedStateDiffRlp, func(i, j int) bool { return receivedStateDiffRlp[i] < receivedStateDiffRlp[j] }) + sort.Slice(expectedStateDiffRlp, func(i, j int) bool { return expectedStateDiffRlp[i] < expectedStateDiffRlp[j] }) + if !bytes.Equal(receivedStateDiffRlp, expectedStateDiffRlp) { + t.Logf("Test failed: %s", test.name) + t.Errorf("actual state diff: %+v\r\n\r\n\r\nexpected state diff: %+v", diff, test.expected) + } + } +} + +func TestBuilderWithRemovedAccountAndStorageWithoutIntermediateNodes(t *testing.T) { + blocks, chain := testhelpers.MakeChain(6, testhelpers.Genesis, testhelpers.TestChainGen) + contractLeafKey = testhelpers.AddressToLeafKey(testhelpers.ContractAddr) + defer chain.Stop() + block3 = blocks[2] + block4 = blocks[3] + block5 = blocks[4] + block6 = blocks[5] + params := statediff.Params{ + IntermediateStateNodes: false, + IntermediateStorageNodes: false, + } + builder = statediff.NewBuilder(chain.StateCache()) + + var tests = []struct { + name string + startingArguments statediff.Args + expected *statediff.StateObject + }{ + // blocks 0-3 are the same as in TestBuilderWithIntermediateNodes + { + "testBlock4", + statediff.Args{ + OldStateRoot: block3.Root(), + NewStateRoot: block4.Root(), + BlockNumber: block4.Number(), + BlockHash: block4.Hash(), + }, + &statediff.StateObject{ + BlockNumber: block4.Number(), + BlockHash: block4.Hash(), + Nodes: []sdtypes.StateNode{ + { + Path: []byte{'\x00'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.BankLeafKey, + NodeValue: bankAccountAtBlock4LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x06'}, + NodeType: sdtypes.Leaf, + LeafKey: contractLeafKey, + NodeValue: contractAccountAtBlock4LeafNode, + StorageNodes: []sdtypes.StorageNode{ + { + Path: []byte{'\x04'}, + NodeType: sdtypes.Leaf, + LeafKey: slot2StorageKey.Bytes(), + NodeValue: slot2StorageLeafNode, + }, + { + Path: []byte{'\x0b'}, + NodeType: sdtypes.Removed, + NodeValue: []byte{}, + }, + { + Path: []byte{'\x0c'}, + NodeType: sdtypes.Removed, + NodeValue: []byte{}, + }, + }, + }, + { + Path: []byte{'\x0c'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.Account2LeafKey, + NodeValue: account2AtBlock4LeafNode, + StorageNodes: emptyStorage, + }, + }, + }, + }, + { + "testBlock5", + statediff.Args{ + OldStateRoot: block4.Root(), + NewStateRoot: block5.Root(), + BlockNumber: block5.Number(), + BlockHash: block5.Hash(), + }, + &statediff.StateObject{ + BlockNumber: block5.Number(), + BlockHash: block5.Hash(), + Nodes: []sdtypes.StateNode{ + { + Path: []byte{'\x00'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.BankLeafKey, + NodeValue: bankAccountAtBlock5LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x06'}, + NodeType: sdtypes.Leaf, + LeafKey: contractLeafKey, + NodeValue: contractAccountAtBlock5LeafNode, + StorageNodes: []sdtypes.StorageNode{ + { + Path: []byte{}, + NodeType: sdtypes.Leaf, + NodeValue: slot0StorageLeafRootNode, + LeafKey: slot0StorageKey.Bytes(), + }, + { + Path: []byte{'\x02'}, + NodeType: sdtypes.Removed, + NodeValue: []byte{}, + }, + { + Path: []byte{'\x04'}, + NodeType: sdtypes.Removed, + NodeValue: []byte{}, + }, + }, + }, + { + Path: []byte{'\x0e'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.Account1LeafKey, + NodeValue: account1AtBlock5LeafNode, + StorageNodes: emptyStorage, + }, + }, + }, + }, + { + "testBlock6", + statediff.Args{ + OldStateRoot: block5.Root(), + NewStateRoot: block6.Root(), + BlockNumber: block6.Number(), + BlockHash: block6.Hash(), + }, + &statediff.StateObject{ + BlockNumber: block6.Number(), + BlockHash: block6.Hash(), + Nodes: []sdtypes.StateNode{ + { + Path: []byte{'\x06'}, + NodeType: sdtypes.Removed, + NodeValue: []byte{}, + }, + { + Path: []byte{'\x0c'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.Account2LeafKey, + NodeValue: account2AtBlock6LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x0e'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.Account1LeafKey, + NodeValue: account1AtBlock6LeafNode, + StorageNodes: emptyStorage, + }, + }, + }, + }, + } + + for _, test := range tests { + diff, err := builder.BuildStateDiffObject(test.startingArguments, params) + if err != nil { + t.Error(err) + } + receivedStateDiffRlp, err := rlp.EncodeToBytes(diff) + if err != nil { + t.Error(err) + } + expectedStateDiffRlp, err := rlp.EncodeToBytes(test.expected) + if err != nil { + t.Error(err) + } + sort.Slice(receivedStateDiffRlp, func(i, j int) bool { return receivedStateDiffRlp[i] < receivedStateDiffRlp[j] }) + sort.Slice(expectedStateDiffRlp, func(i, j int) bool { return expectedStateDiffRlp[i] < expectedStateDiffRlp[j] }) + if !bytes.Equal(receivedStateDiffRlp, expectedStateDiffRlp) { + t.Logf("Test failed: %s", test.name) + t.Errorf("actual state diff: %+v\r\n\r\n\r\nexpected state diff: %+v", diff, test.expected) + } + } +} + +var ( + slot00StorageValue = common.Hex2Bytes("9471562b71999873db5b286df957af199ec94617f7") // prefixed TestBankAddress + + slot00StorageLeafNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("390decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563"), + slot00StorageValue, + }) + + contractAccountAtBlock01, _ = rlp.EncodeToBytes(state.Account{ + Nonce: 1, + Balance: big.NewInt(0), + CodeHash: common.HexToHash("0xaaea5efba4fd7b45d7ec03918ac5d8b31aa93b48986af0e6b591f0f087c80127").Bytes(), + Root: crypto.Keccak256Hash(block01StorageBranchRootNode), + }) + contractAccountAtBlock01LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("3cb2583748c26e89ef19c2a8529b05a270f735553b4d44b6f2a1894987a71c8b"), + contractAccountAtBlock01, + }) + + bankAccountAtBlock01, _ = rlp.EncodeToBytes(state.Account{ + Nonce: 1, + Balance: big.NewInt(testhelpers.TestBankFunds.Int64() + miningReward), + CodeHash: testhelpers.NullCodeHash.Bytes(), + Root: testhelpers.EmptyContractRoot, + }) + bankAccountAtBlock01LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("30bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a"), + bankAccountAtBlock01, + }) + bankAccountAtBlock02, _ = rlp.EncodeToBytes(state.Account{ + Nonce: 2, + Balance: big.NewInt(testhelpers.TestBankFunds.Int64() + miningReward*2), + CodeHash: testhelpers.NullCodeHash.Bytes(), + Root: testhelpers.EmptyContractRoot, + }) + bankAccountAtBlock02LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("2000bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a"), + bankAccountAtBlock02, + }) + + block01BranchRootNode, _ = rlp.EncodeToBytes([]interface{}{ + crypto.Keccak256Hash(bankAccountAtBlock01LeafNode), + crypto.Keccak256Hash(contractAccountAtBlock01LeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + }) + + block01StorageBranchRootNode, _ = rlp.EncodeToBytes([]interface{}{ + []byte{}, + []byte{}, + crypto.Keccak256(slot00StorageLeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + crypto.Keccak256(slot1StorageLeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + }) +) + +func TestBuilderWithMovedAccount(t *testing.T) { + blocks, chain := testhelpers.MakeChain(2, testhelpers.Genesis, testhelpers.TestSelfDestructChainGen) + contractLeafKey = testhelpers.AddressToLeafKey(testhelpers.ContractAddr) + defer chain.Stop() + block0 = testhelpers.Genesis + block1 = blocks[0] + block2 = blocks[1] + params := statediff.Params{ + IntermediateStateNodes: true, + IntermediateStorageNodes: true, + } + builder = statediff.NewBuilder(chain.StateCache()) + + var tests = []struct { + name string + startingArguments statediff.Args + expected *statediff.StateObject + }{ + { + "testBlock1", + statediff.Args{ + OldStateRoot: block0.Root(), + NewStateRoot: block1.Root(), + BlockNumber: block1.Number(), + BlockHash: block1.Hash(), + }, + &statediff.StateObject{ + BlockNumber: block1.Number(), + BlockHash: block1.Hash(), + Nodes: []sdtypes.StateNode{ + { + Path: []byte{}, + NodeType: sdtypes.Branch, + NodeValue: block01BranchRootNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x00'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.BankLeafKey, + NodeValue: bankAccountAtBlock01LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x01'}, + NodeType: sdtypes.Leaf, + LeafKey: contractLeafKey, + NodeValue: contractAccountAtBlock01LeafNode, + StorageNodes: []sdtypes.StorageNode{ + { + Path: []byte{}, + NodeType: sdtypes.Branch, + NodeValue: block01StorageBranchRootNode, + }, + { + Path: []byte{'\x02'}, + NodeType: sdtypes.Leaf, + LeafKey: slot0StorageKey.Bytes(), + NodeValue: slot00StorageLeafNode, + }, + { + Path: []byte{'\x0b'}, + NodeType: sdtypes.Leaf, + LeafKey: slot1StorageKey.Bytes(), + NodeValue: slot1StorageLeafNode, + }, + }, + }, + }, + CodeAndCodeHashes: []sdtypes.CodeAndCodeHash{ + { + Hash: testhelpers.CodeHash, + Code: testhelpers.ByteCodeAfterDeployment, + }, + }, + }, + }, + { + "testBlock2", + statediff.Args{ + OldStateRoot: block1.Root(), + NewStateRoot: block2.Root(), + BlockNumber: block2.Number(), + BlockHash: block2.Hash(), + }, + &statediff.StateObject{ + BlockNumber: block2.Number(), + BlockHash: block2.Hash(), + Nodes: []sdtypes.StateNode{ + { + Path: []byte{}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.BankLeafKey, + NodeValue: bankAccountAtBlock02LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x01'}, + NodeType: sdtypes.Removed, + NodeValue: []byte{}, + }, + { + Path: []byte{'\x00'}, + NodeType: sdtypes.Removed, + NodeValue: []byte{}, + }, + }, + }, + }, + } + + for _, test := range tests { + diff, err := builder.BuildStateDiffObject(test.startingArguments, params) + if err != nil { + t.Error(err) + } + receivedStateDiffRlp, err := rlp.EncodeToBytes(diff) + if err != nil { + t.Error(err) + } + expectedStateDiffRlp, err := rlp.EncodeToBytes(test.expected) + if err != nil { + t.Error(err) + } + sort.Slice(receivedStateDiffRlp, func(i, j int) bool { return receivedStateDiffRlp[i] < receivedStateDiffRlp[j] }) + sort.Slice(expectedStateDiffRlp, func(i, j int) bool { return expectedStateDiffRlp[i] < expectedStateDiffRlp[j] }) + if !bytes.Equal(receivedStateDiffRlp, expectedStateDiffRlp) { + t.Logf("Test failed: %s", test.name) + t.Errorf("actual state diff: %+v\r\n\r\n\r\nexpected state diff: %+v", diff, test.expected) + } + } +} + +func TestBuilderWithMovedAccountOnlyLeafs(t *testing.T) { + blocks, chain := testhelpers.MakeChain(2, testhelpers.Genesis, testhelpers.TestSelfDestructChainGen) + contractLeafKey = testhelpers.AddressToLeafKey(testhelpers.ContractAddr) + defer chain.Stop() + block0 = testhelpers.Genesis + block1 = blocks[0] + block2 = blocks[1] + params := statediff.Params{ + IntermediateStateNodes: false, + IntermediateStorageNodes: false, + } + builder = statediff.NewBuilder(chain.StateCache()) + + var tests = []struct { + name string + startingArguments statediff.Args + expected *statediff.StateObject + }{ + { + "testBlock1", + statediff.Args{ + OldStateRoot: block0.Root(), + NewStateRoot: block1.Root(), + BlockNumber: block1.Number(), + BlockHash: block1.Hash(), + }, + &statediff.StateObject{ + BlockNumber: block1.Number(), + BlockHash: block1.Hash(), + Nodes: []sdtypes.StateNode{ + { + Path: []byte{'\x00'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.BankLeafKey, + NodeValue: bankAccountAtBlock01LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x01'}, + NodeType: sdtypes.Leaf, + LeafKey: contractLeafKey, + NodeValue: contractAccountAtBlock01LeafNode, + StorageNodes: []sdtypes.StorageNode{ + { + Path: []byte{'\x02'}, + NodeType: sdtypes.Leaf, + LeafKey: slot0StorageKey.Bytes(), + NodeValue: slot00StorageLeafNode, + }, + { + Path: []byte{'\x0b'}, + NodeType: sdtypes.Leaf, + LeafKey: slot1StorageKey.Bytes(), + NodeValue: slot1StorageLeafNode, + }, + }, + }, + }, + CodeAndCodeHashes: []sdtypes.CodeAndCodeHash{ + { + Hash: testhelpers.CodeHash, + Code: testhelpers.ByteCodeAfterDeployment, + }, + }, + }, + }, + { + "testBlock2", + statediff.Args{ + OldStateRoot: block1.Root(), + NewStateRoot: block2.Root(), + BlockNumber: block2.Number(), + BlockHash: block2.Hash(), + }, + &statediff.StateObject{ + BlockNumber: block2.Number(), + BlockHash: block2.Hash(), + Nodes: []sdtypes.StateNode{ + { + Path: []byte{}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.BankLeafKey, + NodeValue: bankAccountAtBlock02LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x01'}, + NodeType: sdtypes.Removed, + NodeValue: []byte{}, + }, + { + Path: []byte{'\x00'}, + NodeType: sdtypes.Removed, + NodeValue: []byte{}, + }, + }, + }, + }, + } + + for _, test := range tests { + diff, err := builder.BuildStateDiffObject(test.startingArguments, params) + if err != nil { + t.Error(err) + } + receivedStateDiffRlp, err := rlp.EncodeToBytes(diff) + if err != nil { + t.Error(err) + } + expectedStateDiffRlp, err := rlp.EncodeToBytes(test.expected) + if err != nil { + t.Error(err) + } + sort.Slice(receivedStateDiffRlp, func(i, j int) bool { return receivedStateDiffRlp[i] < receivedStateDiffRlp[j] }) + sort.Slice(expectedStateDiffRlp, func(i, j int) bool { return expectedStateDiffRlp[i] < expectedStateDiffRlp[j] }) + if !bytes.Equal(receivedStateDiffRlp, expectedStateDiffRlp) { + t.Logf("Test failed: %s", test.name) + t.Errorf("actual state diff: %+v\r\n\r\n\r\nexpected state diff: %+v", diff, test.expected) + } + } +} + +func TestBuildStateTrie(t *testing.T) { + blocks, chain := testhelpers.MakeChain(3, testhelpers.Genesis, testhelpers.TestChainGen) + contractLeafKey = testhelpers.AddressToLeafKey(testhelpers.ContractAddr) + defer chain.Stop() + block1 = blocks[0] + block2 = blocks[1] + block3 = blocks[2] + builder = statediff.NewBuilder(chain.StateCache()) + + var tests = []struct { + name string + block *types.Block + expected *statediff.StateObject + }{ + { + "testBlock1", + block1, + &statediff.StateObject{ + BlockNumber: block1.Number(), + BlockHash: block1.Hash(), + Nodes: []sdtypes.StateNode{ + { + Path: []byte{}, + NodeType: sdtypes.Branch, + NodeValue: block1BranchRootNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x00'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.BankLeafKey, + NodeValue: bankAccountAtBlock1LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x05'}, + NodeType: sdtypes.Leaf, + LeafKey: minerLeafKey, + NodeValue: minerAccountAtBlock1LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x0e'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.Account1LeafKey, + NodeValue: account1AtBlock1LeafNode, + StorageNodes: emptyStorage, + }, + }, + }, + }, + { + "testBlock2", + block2, + &statediff.StateObject{ + BlockNumber: block2.Number(), + BlockHash: block2.Hash(), + Nodes: []sdtypes.StateNode{ + { + Path: []byte{}, + NodeType: sdtypes.Branch, + NodeValue: block2BranchRootNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x00'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.BankLeafKey, + NodeValue: bankAccountAtBlock2LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x05'}, + NodeType: sdtypes.Leaf, + LeafKey: minerLeafKey, + NodeValue: minerAccountAtBlock2LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x0e'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.Account1LeafKey, + NodeValue: account1AtBlock2LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x06'}, + NodeType: sdtypes.Leaf, + LeafKey: contractLeafKey, + NodeValue: contractAccountAtBlock2LeafNode, + StorageNodes: []sdtypes.StorageNode{ + { + Path: []byte{}, + NodeType: sdtypes.Branch, + NodeValue: block2StorageBranchRootNode, + }, + { + Path: []byte{'\x02'}, + NodeType: sdtypes.Leaf, + LeafKey: slot0StorageKey.Bytes(), + NodeValue: slot0StorageLeafNode, + }, + { + Path: []byte{'\x0b'}, + NodeType: sdtypes.Leaf, + LeafKey: slot1StorageKey.Bytes(), + NodeValue: slot1StorageLeafNode, + }, + }, + }, + { + Path: []byte{'\x0c'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.Account2LeafKey, + NodeValue: account2AtBlock2LeafNode, + StorageNodes: emptyStorage, + }, + }, + CodeAndCodeHashes: []sdtypes.CodeAndCodeHash{ + { + Hash: testhelpers.CodeHash, + Code: testhelpers.ByteCodeAfterDeployment, + }, + }, + }, + }, + { + "testBlock3", + block3, + &statediff.StateObject{ + BlockNumber: block3.Number(), + BlockHash: block3.Hash(), + Nodes: []sdtypes.StateNode{ + { + Path: []byte{}, + NodeType: sdtypes.Branch, + NodeValue: block3BranchRootNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x00'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.BankLeafKey, + NodeValue: bankAccountAtBlock3LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x05'}, + NodeType: sdtypes.Leaf, + LeafKey: minerLeafKey, + NodeValue: minerAccountAtBlock2LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x0e'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.Account1LeafKey, + NodeValue: account1AtBlock2LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x06'}, + NodeType: sdtypes.Leaf, + LeafKey: contractLeafKey, + NodeValue: contractAccountAtBlock3LeafNode, + StorageNodes: []sdtypes.StorageNode{ + { + Path: []byte{}, + NodeType: sdtypes.Branch, + NodeValue: block3StorageBranchRootNode, + }, + { + Path: []byte{'\x02'}, + NodeType: sdtypes.Leaf, + LeafKey: slot0StorageKey.Bytes(), + NodeValue: slot0StorageLeafNode, + }, + { + Path: []byte{'\x0b'}, + NodeType: sdtypes.Leaf, + LeafKey: slot1StorageKey.Bytes(), + NodeValue: slot1StorageLeafNode, + }, + { + Path: []byte{'\x0c'}, + NodeType: sdtypes.Leaf, + LeafKey: slot3StorageKey.Bytes(), + NodeValue: slot3StorageLeafNode, + }, + }, + }, + { + Path: []byte{'\x0c'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.Account2LeafKey, + NodeValue: account2AtBlock3LeafNode, + StorageNodes: emptyStorage, + }, + }, + CodeAndCodeHashes: []sdtypes.CodeAndCodeHash{ + { + Hash: testhelpers.CodeHash, + Code: testhelpers.ByteCodeAfterDeployment, + }, + }, + }, + }, + } + + for _, test := range tests { + diff, err := builder.BuildStateTrieObject(test.block) + if err != nil { + t.Error(err) + } + receivedStateTrieRlp, err := rlp.EncodeToBytes(diff) + if err != nil { + t.Error(err) + } + expectedStateTrieRlp, err := rlp.EncodeToBytes(test.expected) + if err != nil { + t.Error(err) + } + sort.Slice(receivedStateTrieRlp, func(i, j int) bool { return receivedStateTrieRlp[i] < receivedStateTrieRlp[j] }) + sort.Slice(expectedStateTrieRlp, func(i, j int) bool { return expectedStateTrieRlp[i] < expectedStateTrieRlp[j] }) + if !bytes.Equal(receivedStateTrieRlp, expectedStateTrieRlp) { + t.Logf("Test failed: %s", test.name) + t.Errorf("actual state trie: %+v\r\n\r\n\r\nexpected state trie: %+v", diff, test.expected) + } + } +} + +/* +pragma solidity ^0.5.10; + +contract test { + address payable owner; + + modifier onlyOwner { + require( + msg.sender == owner, + "Only owner can call this function." + ); + _; + } + + uint256[100] data; + + constructor() public { + owner = msg.sender; + data = [1]; + } + + function Put(uint256 addr, uint256 value) public { + data[addr] = value; + } + + function close() public onlyOwner { //onlyOwner is custom modifier + selfdestruct(owner); // `owner` is the owners address + } +} +*/ diff --git a/statediff/doc.go b/statediff/doc.go new file mode 100644 index 000000000..184bc242c --- /dev/null +++ b/statediff/doc.go @@ -0,0 +1,57 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +/* +Package statediff provides an auxiliary service that processes state diff objects from incoming chain events, +relaying the objects to any rpc subscriptions. + +This work is adapted from work by Charles Crain at https://github.com/jpmorganchase/quorum/blob/9b7fd9af8082795eeeb6863d9746f12b82dd5078/statediff/statediff.go + +The service is spun up using the below CLI flags +--statediff: boolean flag, turns on the service +--statediff.streamblock: boolean flag, configures the service to associate and stream out the rest of the block data with the state diffs. +--statediff.intermediatenodes: boolean flag, tells service to include intermediate (branch and extension) nodes; default (false) processes leaf nodes only. +--statediff.watchedaddresses: string slice flag, used to limit the state diffing process to the given addresses. Usage: --statediff.watchedaddresses=addr1 --statediff.watchedaddresses=addr2 --statediff.watchedaddresses=addr3 + +If you wish to use the websocket endpoint to subscribe to the statediff service, be sure to open up the Websocket RPC server with the `--ws` flag. The IPC-RPC server is turned on by default. + +The statediffing services works only with `--syncmode="full", but -importantly- does not require garbage collection to be turned off (does not require an archival node). + +e.g. + +$ ./geth --statediff --statediff.streamblock --ws --syncmode "full" + +This starts up the geth node in full sync mode, starts up the statediffing service, and opens up the websocket endpoint to subscribe to the service. +Because the "streamblock" flag has been turned on, the service will strean out block data (headers, transactions, and receipts) along with the diffed state and storage leafs. + +Rpc subscriptions to the service can be created using the rpc.Client.Subscribe() method, +with the "statediff" namespace, a statediff.Payload channel, and the name of the statediff api's rpc method- "stream". + +e.g. + +cli, _ := rpc.Dial("ipcPathOrWsURL") +stateDiffPayloadChan := make(chan statediff.Payload, 20000) +rpcSub, err := cli.Subscribe(context.Background(), "statediff", stateDiffPayloadChan, "stream"}) +for { + select { + case stateDiffPayload := <- stateDiffPayloadChan: + processPayload(stateDiffPayload) + case err := <- rpcSub.Err(): + log.Error(err) + } +} +*/ +package statediff diff --git a/statediff/helpers.go b/statediff/helpers.go new file mode 100644 index 000000000..51ac5c1be --- /dev/null +++ b/statediff/helpers.go @@ -0,0 +1,98 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package statediff + +import ( + "fmt" + "sort" + "strings" + + sdtypes "github.com/ethereum/go-ethereum/statediff/types" +) + +func sortKeys(data AccountMap) []string { + keys := make([]string, 0, len(data)) + for key := range data { + keys = append(keys, key) + } + sort.Strings(keys) + + return keys +} + +// findIntersection finds the set of strings from both arrays that are equivalent +// a and b must first be sorted +// this is used to find which keys have been both "deleted" and "created" i.e. they were updated +func findIntersection(a, b []string) []string { + lenA := len(a) + lenB := len(b) + iOfA, iOfB := 0, 0 + updates := make([]string, 0) + if iOfA >= lenA || iOfB >= lenB { + return updates + } + for { + switch strings.Compare(a[iOfA], b[iOfB]) { + // -1 when a[iOfA] < b[iOfB] + case -1: + iOfA++ + if iOfA >= lenA { + return updates + } + // 0 when a[iOfA] == b[iOfB] + case 0: + updates = append(updates, a[iOfA]) + iOfA++ + iOfB++ + if iOfA >= lenA || iOfB >= lenB { + return updates + } + // 1 when a[iOfA] > b[iOfB] + case 1: + iOfB++ + if iOfB >= lenB { + return updates + } + } + } + +} + +// CheckKeyType checks what type of key we have +func CheckKeyType(elements []interface{}) (sdtypes.NodeType, error) { + if len(elements) > 2 { + return sdtypes.Branch, nil + } + if len(elements) < 2 { + return sdtypes.Unknown, fmt.Errorf("node cannot be less than two elements in length") + } + switch elements[0].([]byte)[0] / 16 { + case '\x00': + return sdtypes.Extension, nil + case '\x01': + return sdtypes.Extension, nil + case '\x02': + return sdtypes.Leaf, nil + case '\x03': + return sdtypes.Leaf, nil + default: + return sdtypes.Unknown, fmt.Errorf("unknown hex prefix") + } +} diff --git a/statediff/indexer/helpers.go b/statediff/indexer/helpers.go new file mode 100644 index 000000000..bb62fd079 --- /dev/null +++ b/statediff/indexer/helpers.go @@ -0,0 +1,55 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package indexer + +import ( + "fmt" + + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/statediff/types" +) + +func ResolveFromNodeType(nodeType types.NodeType) int { + switch nodeType { + case types.Branch: + return 0 + case types.Extension: + return 1 + case types.Leaf: + return 2 + case types.Removed: + return 3 + default: + return -1 + } +} + +// ChainConfig returns the appropriate ethereum chain config for the provided chain id +func ChainConfig(chainID uint64) (*params.ChainConfig, error) { + switch chainID { + case 1: + return params.MainnetChainConfig, nil + case 3: + return params.RopstenChainConfig, nil + case 4: + return params.RinkebyChainConfig, nil + case 5: + return params.GoerliChainConfig, nil + default: + return nil, fmt.Errorf("chain config for chainid %d not available", chainID) + } +} diff --git a/statediff/indexer/indexer.go b/statediff/indexer/indexer.go new file mode 100644 index 000000000..2a6bac3f2 --- /dev/null +++ b/statediff/indexer/indexer.go @@ -0,0 +1,423 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +// This package provides an interface for pushing and indexing IPLD objects into a Postgres database +// Metrics for reporting processing and connection stats are defined in ./metrics.go +package indexer + +import ( + "fmt" + "math/big" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rlp" + + node "github.com/ipfs/go-ipld-format" + "github.com/jmoiron/sqlx" + "github.com/multiformats/go-multihash" + + "github.com/ethereum/go-ethereum/statediff/indexer/ipfs/ipld" + "github.com/ethereum/go-ethereum/statediff/indexer/models" + "github.com/ethereum/go-ethereum/statediff/indexer/postgres" + "github.com/ethereum/go-ethereum/statediff/indexer/shared" + sdtypes "github.com/ethereum/go-ethereum/statediff/types" +) + +var ( + indexerMetrics = RegisterIndexerMetrics(metrics.DefaultRegistry) + dbMetrics = RegisterDBMetrics(metrics.DefaultRegistry) +) + +// Indexer interface to allow substitution of mocks for testing +type Indexer interface { + PushBlock(block *types.Block, receipts types.Receipts, totalDifficulty *big.Int) (*BlockTx, error) + PushStateNode(tx *BlockTx, stateNode sdtypes.StateNode) error + PushCodeAndCodeHash(tx *BlockTx, codeAndCodeHash sdtypes.CodeAndCodeHash) error + ReportDBMetrics(delay time.Duration, quit <-chan bool) +} + +// StateDiffIndexer satisfies the Indexer interface for ethereum statediff objects +type StateDiffIndexer struct { + chainConfig *params.ChainConfig + dbWriter *PostgresCIDWriter +} + +// NewStateDiffIndexer creates a pointer to a new PayloadConverter which satisfies the PayloadConverter interface +func NewStateDiffIndexer(chainConfig *params.ChainConfig, db *postgres.DB) *StateDiffIndexer { + return &StateDiffIndexer{ + chainConfig: chainConfig, + dbWriter: NewPostgresCIDWriter(db), + } +} + +type BlockTx struct { + dbtx *sqlx.Tx + BlockNumber uint64 + headerID int64 + err error + Close func() error +} + +// Reporting function to run as goroutine +func (sdi *StateDiffIndexer) ReportDBMetrics(delay time.Duration, quit <-chan bool) { + if !metrics.Enabled { + return + } + ticker := time.NewTicker(delay) + go func() { + for { + select { + case <-ticker.C: + dbMetrics.Update(sdi.dbWriter.db.Stats()) + case <-quit: + ticker.Stop() + return + } + } + }() +} + +// Pushes and indexes block data in database, except state & storage nodes (includes header, uncles, transactions & receipts) +// Returns an initiated DB transaction which must be Closed via defer to commit or rollback +func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receipts, totalDifficulty *big.Int) (*BlockTx, error) { + start, t := time.Now(), time.Now() + blockHash := block.Hash() + blockHashStr := blockHash.String() + height := block.NumberU64() + traceMsg := fmt.Sprintf("indexer stats for statediff at %d with hash %s:\r\n", height, blockHashStr) + transactions := block.Transactions() + // Derive any missing fields + if err := receipts.DeriveFields(sdi.chainConfig, blockHash, height, transactions); err != nil { + return nil, err + } + // Generate the block iplds + headerNode, uncleNodes, txNodes, txTrieNodes, rctNodes, rctTrieNodes, err := ipld.FromBlockAndReceipts(block, receipts) + if err != nil { + return nil, err + } + if len(txNodes) != len(txTrieNodes) && len(rctNodes) != len(rctTrieNodes) && len(txNodes) != len(rctNodes) { + return nil, fmt.Errorf("expected number of transactions (%d), transaction trie nodes (%d), receipts (%d), and receipt trie nodes (%d)to be equal", len(txNodes), len(txTrieNodes), len(rctNodes), len(rctTrieNodes)) + } + // Calculate reward + reward := CalcEthBlockReward(block.Header(), block.Uncles(), block.Transactions(), receipts) + t = time.Now() + // Begin new db tx for everything + tx, err := sdi.dbWriter.db.Beginx() + if err != nil { + return nil, err + } + blocktx := BlockTx{ + dbtx: tx, + // handle transaction commit or rollback for any return case + Close: func() error { + var err error + if p := recover(); p != nil { + shared.Rollback(tx) + panic(p) + } else { + tDiff := time.Now().Sub(t) + indexerMetrics.tStateStoreCodeProcessing.Update(tDiff) + traceMsg += fmt.Sprintf("state, storage, and code storage processing time: %s\r\n", tDiff.String()) + t = time.Now() + err = tx.Commit() + tDiff = time.Now().Sub(t) + indexerMetrics.tPostgresCommit.Update(tDiff) + traceMsg += fmt.Sprintf("postgres transaction commit duration: %s\r\n", tDiff.String()) + } + traceMsg += fmt.Sprintf(" TOTAL PROCESSING DURATION: %s\r\n", time.Now().Sub(start).String()) + log.Debug(traceMsg) + return err + }, + } + tDiff := time.Now().Sub(t) + indexerMetrics.tFreePostgres.Update(tDiff) + + traceMsg += fmt.Sprintf("time spent waiting for free postgres tx: %s:\r\n", tDiff.String()) + t = time.Now() + + // Publish and index header, collect headerID + headerID, err := sdi.processHeader(tx, block.Header(), headerNode, reward, totalDifficulty) + if err != nil { + return nil, err + } + tDiff = time.Now().Sub(t) + indexerMetrics.tHeaderProcessing.Update(tDiff) + traceMsg += fmt.Sprintf("header processing time: %s\r\n", tDiff.String()) + t = time.Now() + // Publish and index uncles + if err := sdi.processUncles(tx, headerID, height, uncleNodes); err != nil { + return nil, err + } + tDiff = time.Now().Sub(t) + indexerMetrics.tUncleProcessing.Update(tDiff) + traceMsg += fmt.Sprintf("uncle processing time: %s\r\n", tDiff.String()) + t = time.Now() + // Publish and index receipts and txs + if err := sdi.processReceiptsAndTxs(tx, processArgs{ + headerID: headerID, + blockNumber: block.Number(), + receipts: receipts, + txs: transactions, + rctNodes: rctNodes, + rctTrieNodes: rctTrieNodes, + txNodes: txNodes, + txTrieNodes: txTrieNodes, + }); err != nil { + return nil, err + } + tDiff = time.Now().Sub(t) + indexerMetrics.tTxAndRecProcessing.Update(tDiff) + traceMsg += fmt.Sprintf("tx and receipt processing time: %s\r\n", tDiff.String()) + t = time.Now() + + blocktx.BlockNumber = height + blocktx.headerID = headerID + return &blocktx, err +} + +// processHeader publishes and indexes a header IPLD in Postgres +// it returns the headerID +func (sdi *StateDiffIndexer) processHeader(tx *sqlx.Tx, header *types.Header, headerNode node.Node, reward, td *big.Int) (int64, error) { + // publish header + if err := shared.PublishIPLD(tx, headerNode); err != nil { + return 0, err + } + // index header + return sdi.dbWriter.upsertHeaderCID(tx, models.HeaderModel{ + CID: headerNode.Cid().String(), + MhKey: shared.MultihashKeyFromCID(headerNode.Cid()), + ParentHash: header.ParentHash.String(), + BlockNumber: header.Number.String(), + BlockHash: header.Hash().String(), + TotalDifficulty: td.String(), + Reward: reward.String(), + Bloom: header.Bloom.Bytes(), + StateRoot: header.Root.String(), + RctRoot: header.ReceiptHash.String(), + TxRoot: header.TxHash.String(), + UncleRoot: header.UncleHash.String(), + Timestamp: header.Time, + }) +} + +func (sdi *StateDiffIndexer) processUncles(tx *sqlx.Tx, headerID int64, blockNumber uint64, uncleNodes []*ipld.EthHeader) error { + // publish and index uncles + for _, uncleNode := range uncleNodes { + if err := shared.PublishIPLD(tx, uncleNode); err != nil { + return err + } + uncleReward := CalcUncleMinerReward(blockNumber, uncleNode.Number.Uint64()) + uncle := models.UncleModel{ + CID: uncleNode.Cid().String(), + MhKey: shared.MultihashKeyFromCID(uncleNode.Cid()), + ParentHash: uncleNode.ParentHash.String(), + BlockHash: uncleNode.Hash().String(), + Reward: uncleReward.String(), + } + if err := sdi.dbWriter.upsertUncleCID(tx, uncle, headerID); err != nil { + return err + } + } + return nil +} + +// processArgs bundles arugments to processReceiptsAndTxs +type processArgs struct { + headerID int64 + blockNumber *big.Int + receipts types.Receipts + txs types.Transactions + rctNodes []*ipld.EthReceipt + rctTrieNodes []*ipld.EthRctTrie + txNodes []*ipld.EthTx + txTrieNodes []*ipld.EthTxTrie +} + +// processReceiptsAndTxs publishes and indexes receipt and transaction IPLDs in Postgres +func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *sqlx.Tx, args processArgs) error { + // Process receipts and txs + signer := types.MakeSigner(sdi.chainConfig, args.blockNumber) + for i, receipt := range args.receipts { + // tx that corresponds with this receipt + trx := args.txs[i] + from, err := types.Sender(signer, trx) + if err != nil { + return err + } + + // Publishing + // publish trie nodes, these aren't indexed directly + if err := shared.PublishIPLD(tx, args.txTrieNodes[i]); err != nil { + return err + } + if err := shared.PublishIPLD(tx, args.rctTrieNodes[i]); err != nil { + return err + } + // publish the txs and receipts + txNode, rctNode := args.txNodes[i], args.rctNodes[i] + if err := shared.PublishIPLD(tx, txNode); err != nil { + return err + } + if err := shared.PublishIPLD(tx, rctNode); err != nil { + return err + } + + // Indexing + // extract topic and contract data from the receipt for indexing + topicSets := make([][]string, 4) + mappedContracts := make(map[string]bool) // use map to avoid duplicate addresses + for _, log := range receipt.Logs { + for i, topic := range log.Topics { + topicSets[i] = append(topicSets[i], topic.Hex()) + } + mappedContracts[log.Address.String()] = true + } + // these are the contracts seen in the logs + logContracts := make([]string, 0, len(mappedContracts)) + for addr := range mappedContracts { + logContracts = append(logContracts, addr) + } + // this is the contract address if this receipt is for a contract creation tx + contract := shared.HandleZeroAddr(receipt.ContractAddress) + var contractHash string + isDeployment := contract != "" + if isDeployment { + contractHash = crypto.Keccak256Hash(common.HexToAddress(contract).Bytes()).String() + // if tx is a contract deployment, publish the data (code) + // codec doesn't matter in this case sine we are not interested in the cid and the db key is multihash-derived + // TODO: THE DATA IS NOT DIRECTLY THE CONTRACT CODE; THERE IS A MISSING PROCESSING STEP HERE + // the contractHash => contract code is not currently correct + if _, err := shared.PublishRaw(tx, ipld.MEthStorageTrie, multihash.KECCAK_256, trx.Data()); err != nil { + return err + } + } + // index tx first so that the receipt can reference it by FK + txModel := models.TxModel{ + Dst: shared.HandleZeroAddrPointer(trx.To()), + Src: shared.HandleZeroAddr(from), + TxHash: trx.Hash().String(), + Index: int64(i), + Data: trx.Data(), + Deployment: isDeployment, + CID: txNode.Cid().String(), + MhKey: shared.MultihashKeyFromCID(txNode.Cid()), + } + txID, err := sdi.dbWriter.upsertTransactionCID(tx, txModel, args.headerID) + if err != nil { + return err + } + // index the receipt + rctModel := models.ReceiptModel{ + Topic0s: topicSets[0], + Topic1s: topicSets[1], + Topic2s: topicSets[2], + Topic3s: topicSets[3], + Contract: contract, + ContractHash: contractHash, + LogContracts: logContracts, + CID: rctNode.Cid().String(), + MhKey: shared.MultihashKeyFromCID(rctNode.Cid()), + } + if err := sdi.dbWriter.upsertReceiptCID(tx, rctModel, txID); err != nil { + return err + } + } + return nil +} + +func (sdi *StateDiffIndexer) PushStateNode(tx *BlockTx, stateNode sdtypes.StateNode) error { + // publish the state node + stateCIDStr, err := shared.PublishRaw(tx.dbtx, ipld.MEthStateTrie, multihash.KECCAK_256, stateNode.NodeValue) + if err != nil { + return err + } + mhKey, _ := shared.MultihashKeyFromCIDString(stateCIDStr) + stateModel := models.StateNodeModel{ + Path: stateNode.Path, + StateKey: common.BytesToHash(stateNode.LeafKey).String(), + CID: stateCIDStr, + MhKey: mhKey, + NodeType: ResolveFromNodeType(stateNode.NodeType), + } + // index the state node, collect the stateID to reference by FK + stateID, err := sdi.dbWriter.upsertStateCID(tx.dbtx, stateModel, tx.headerID) + if err != nil { + return err + } + // if we have a leaf, decode and index the account data + if stateNode.NodeType == sdtypes.Leaf { + var i []interface{} + if err := rlp.DecodeBytes(stateNode.NodeValue, &i); err != nil { + return fmt.Errorf("error decoding state leaf node rlp: %s", err.Error()) + } + if len(i) != 2 { + return fmt.Errorf("eth IPLDPublisher expected state leaf node rlp to decode into two elements") + } + var account state.Account + if err := rlp.DecodeBytes(i[1].([]byte), &account); err != nil { + return fmt.Errorf("error decoding state account rlp: %s", err.Error()) + } + accountModel := models.StateAccountModel{ + Balance: account.Balance.String(), + Nonce: account.Nonce, + CodeHash: account.CodeHash, + StorageRoot: account.Root.String(), + } + if err := sdi.dbWriter.upsertStateAccount(tx.dbtx, accountModel, stateID); err != nil { + return err + } + } + // if there are any storage nodes associated with this node, publish and index them + for _, storageNode := range stateNode.StorageNodes { + storageCIDStr, err := shared.PublishRaw(tx.dbtx, ipld.MEthStorageTrie, multihash.KECCAK_256, storageNode.NodeValue) + if err != nil { + return err + } + mhKey, _ := shared.MultihashKeyFromCIDString(storageCIDStr) + storageModel := models.StorageNodeModel{ + Path: storageNode.Path, + StorageKey: common.BytesToHash(storageNode.LeafKey).String(), + CID: storageCIDStr, + MhKey: mhKey, + NodeType: ResolveFromNodeType(storageNode.NodeType), + } + if err := sdi.dbWriter.upsertStorageCID(tx.dbtx, storageModel, stateID); err != nil { + return err + } + } + + return nil +} + +// Publishes code and codehash pairs to the ipld database +func (sdi *StateDiffIndexer) PushCodeAndCodeHash(tx *BlockTx, codeAndCodeHash sdtypes.CodeAndCodeHash) error { + // codec doesn't matter since db key is multihash-based + mhKey, err := shared.MultihashKeyFromKeccak256(codeAndCodeHash.Hash) + if err != nil { + return err + } + if err := shared.PublishDirect(tx.dbtx, mhKey, codeAndCodeHash.Code); err != nil { + return err + } + return nil +} diff --git a/statediff/indexer/indexer_test.go b/statediff/indexer/indexer_test.go new file mode 100644 index 000000000..1a7c76763 --- /dev/null +++ b/statediff/indexer/indexer_test.go @@ -0,0 +1,292 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package indexer_test + +import ( + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/params" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-ipfs-blockstore" + "github.com/ipfs/go-ipfs-ds-help" + + ind "github.com/ethereum/go-ethereum/statediff/indexer" + "github.com/ethereum/go-ethereum/statediff/indexer/mocks" + eth "github.com/ethereum/go-ethereum/statediff/indexer/models" + "github.com/ethereum/go-ethereum/statediff/indexer/postgres" + "github.com/ethereum/go-ethereum/statediff/indexer/shared" +) + +var ( + db *postgres.DB + err error + indexer *ind.StateDiffIndexer + ipfsPgGet = `SELECT data FROM public.blocks + WHERE key = $1` +) + +func expectTrue(t *testing.T, value bool) { + if !value { + t.Fatalf("Assertion failed") + } +} + +func setup(t *testing.T) { + db, err = shared.SetupDB() + if err != nil { + t.Fatal(err) + } + indexer = ind.NewStateDiffIndexer(params.MainnetChainConfig, db) + var tx *ind.BlockTx + tx, err = indexer.PushBlock( + mocks.MockBlock, + mocks.MockReceipts, + mocks.MockBlock.Difficulty()) + if err != nil { + t.Fatal(err) + } + defer tx.Close() + for _, node := range mocks.StateDiffs { + err = indexer.PushStateNode(tx, node) + if err != nil { + t.Fatal(err) + } + } + + shared.ExpectEqual(t, tx.BlockNumber, mocks.BlockNumber.Uint64()) +} + +func tearDown(t *testing.T) { + ind.TearDownDB(t, db) +} + +func TestPublishAndIndexer(t *testing.T) { + t.Run("Publish and index header IPLDs in a single tx", func(t *testing.T) { + setup(t) + defer tearDown(t) + pgStr := `SELECT cid, td, reward, id + FROM eth.header_cids + WHERE block_number = $1` + // check header was properly indexed + type res struct { + CID string + TD string + Reward string + ID int + } + header := new(res) + err = db.QueryRowx(pgStr, 1).StructScan(header) + if err != nil { + t.Fatal(err) + } + shared.ExpectEqual(t, header.CID, mocks.HeaderCID.String()) + shared.ExpectEqual(t, header.TD, mocks.MockBlock.Difficulty().String()) + shared.ExpectEqual(t, header.Reward, "5000000000000011250") + dc, err := cid.Decode(header.CID) + if err != nil { + t.Fatal(err) + } + mhKey := dshelp.MultihashToDsKey(dc.Hash()) + prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() + var data []byte + err = db.Get(&data, ipfsPgGet, prefixedKey) + if err != nil { + t.Fatal(err) + } + shared.ExpectEqual(t, data, mocks.MockHeaderRlp) + }) + + t.Run("Publish and index transaction IPLDs in a single tx", func(t *testing.T) { + setup(t) + defer tearDown(t) + // check that txs were properly indexed + trxs := make([]string, 0) + pgStr := `SELECT transaction_cids.cid FROM eth.transaction_cids INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id) + WHERE header_cids.block_number = $1` + err = db.Select(&trxs, pgStr, 1) + if err != nil { + t.Fatal(err) + } + shared.ExpectEqual(t, len(trxs), 3) + expectTrue(t, shared.ListContainsString(trxs, mocks.Trx1CID.String())) + expectTrue(t, shared.ListContainsString(trxs, mocks.Trx2CID.String())) + expectTrue(t, shared.ListContainsString(trxs, mocks.Trx3CID.String())) + // and published + for _, c := range trxs { + dc, err := cid.Decode(c) + if err != nil { + t.Fatal(err) + } + mhKey := dshelp.MultihashToDsKey(dc.Hash()) + prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() + var data []byte + err = db.Get(&data, ipfsPgGet, prefixedKey) + if err != nil { + t.Fatal(err) + } + switch c { + case mocks.Trx1CID.String(): + shared.ExpectEqual(t, data, mocks.MockTransactions.GetRlp(0)) + case mocks.Trx2CID.String(): + shared.ExpectEqual(t, data, mocks.MockTransactions.GetRlp(1)) + case mocks.Trx3CID.String(): + shared.ExpectEqual(t, data, mocks.MockTransactions.GetRlp(2)) + } + } + }) + + t.Run("Publish and index receipt IPLDs in a single tx", func(t *testing.T) { + setup(t) + defer tearDown(t) + // check receipts were properly indexed + rcts := make([]string, 0) + pgStr := `SELECT receipt_cids.cid FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids + WHERE receipt_cids.tx_id = transaction_cids.id + AND transaction_cids.header_id = header_cids.id + AND header_cids.block_number = $1` + err = db.Select(&rcts, pgStr, 1) + if err != nil { + t.Fatal(err) + } + shared.ExpectEqual(t, len(rcts), 3) + expectTrue(t, shared.ListContainsString(rcts, mocks.Rct1CID.String())) + expectTrue(t, shared.ListContainsString(rcts, mocks.Rct2CID.String())) + expectTrue(t, shared.ListContainsString(rcts, mocks.Rct3CID.String())) + // and published + for _, c := range rcts { + dc, err := cid.Decode(c) + if err != nil { + t.Fatal(err) + } + mhKey := dshelp.MultihashToDsKey(dc.Hash()) + prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() + var data []byte + err = db.Get(&data, ipfsPgGet, prefixedKey) + if err != nil { + t.Fatal(err) + } + switch c { + case mocks.Rct1CID.String(): + shared.ExpectEqual(t, data, mocks.MockReceipts.GetRlp(0)) + case mocks.Rct2CID.String(): + shared.ExpectEqual(t, data, mocks.MockReceipts.GetRlp(1)) + case mocks.Rct3CID.String(): + shared.ExpectEqual(t, data, mocks.MockReceipts.GetRlp(2)) + } + } + }) + + t.Run("Publish and index state IPLDs in a single tx", func(t *testing.T) { + setup(t) + defer tearDown(t) + // check that state nodes were properly indexed and published + stateNodes := make([]eth.StateNodeModel, 0) + pgStr := `SELECT state_cids.id, state_cids.cid, state_cids.state_leaf_key, state_cids.node_type, state_cids.state_path, state_cids.header_id + FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) + WHERE header_cids.block_number = $1` + err = db.Select(&stateNodes, pgStr, 1) + if err != nil { + t.Fatal(err) + } + shared.ExpectEqual(t, len(stateNodes), 2) + for _, stateNode := range stateNodes { + var data []byte + dc, err := cid.Decode(stateNode.CID) + if err != nil { + t.Fatal(err) + } + mhKey := dshelp.MultihashToDsKey(dc.Hash()) + prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() + err = db.Get(&data, ipfsPgGet, prefixedKey) + if err != nil { + t.Fatal(err) + } + pgStr = `SELECT * from eth.state_accounts WHERE state_id = $1` + var account eth.StateAccountModel + err = db.Get(&account, pgStr, stateNode.ID) + if err != nil { + t.Fatal(err) + } + if stateNode.CID == mocks.State1CID.String() { + shared.ExpectEqual(t, stateNode.NodeType, 2) + shared.ExpectEqual(t, stateNode.StateKey, common.BytesToHash(mocks.ContractLeafKey).Hex()) + shared.ExpectEqual(t, stateNode.Path, []byte{'\x06'}) + shared.ExpectEqual(t, data, mocks.ContractLeafNode) + shared.ExpectEqual(t, account, eth.StateAccountModel{ + ID: account.ID, + StateID: stateNode.ID, + Balance: "0", + CodeHash: mocks.ContractCodeHash.Bytes(), + StorageRoot: mocks.ContractRoot, + Nonce: 1, + }) + } + if stateNode.CID == mocks.State2CID.String() { + shared.ExpectEqual(t, stateNode.NodeType, 2) + shared.ExpectEqual(t, stateNode.StateKey, common.BytesToHash(mocks.AccountLeafKey).Hex()) + shared.ExpectEqual(t, stateNode.Path, []byte{'\x0c'}) + shared.ExpectEqual(t, data, mocks.AccountLeafNode) + shared.ExpectEqual(t, account, eth.StateAccountModel{ + ID: account.ID, + StateID: stateNode.ID, + Balance: "1000", + CodeHash: mocks.AccountCodeHash.Bytes(), + StorageRoot: mocks.AccountRoot, + Nonce: 0, + }) + } + } + pgStr = `SELECT * from eth.state_accounts WHERE state_id = $1` + }) + + t.Run("Publish and index storage IPLDs in a single tx", func(t *testing.T) { + setup(t) + defer tearDown(t) + // check that storage nodes were properly indexed + storageNodes := make([]eth.StorageNodeWithStateKeyModel, 0) + pgStr := `SELECT storage_cids.cid, state_cids.state_leaf_key, storage_cids.storage_leaf_key, storage_cids.node_type, storage_cids.storage_path + FROM eth.storage_cids, eth.state_cids, eth.header_cids + WHERE storage_cids.state_id = state_cids.id + AND state_cids.header_id = header_cids.id + AND header_cids.block_number = $1` + err = db.Select(&storageNodes, pgStr, 1) + if err != nil { + t.Fatal(err) + } + shared.ExpectEqual(t, len(storageNodes), 1) + shared.ExpectEqual(t, storageNodes[0], eth.StorageNodeWithStateKeyModel{ + CID: mocks.StorageCID.String(), + NodeType: 2, + StorageKey: common.BytesToHash(mocks.StorageLeafKey).Hex(), + StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(), + Path: []byte{}, + }) + var data []byte + dc, err := cid.Decode(storageNodes[0].CID) + if err != nil { + t.Fatal(err) + } + mhKey := dshelp.MultihashToDsKey(dc.Hash()) + prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() + err = db.Get(&data, ipfsPgGet, prefixedKey) + if err != nil { + t.Fatal(err) + } + shared.ExpectEqual(t, data, mocks.StorageLeafNode) + }) +} diff --git a/statediff/indexer/ipfs/ipld/eth_account.go b/statediff/indexer/ipfs/ipld/eth_account.go new file mode 100644 index 000000000..5d80af1d9 --- /dev/null +++ b/statediff/indexer/ipfs/ipld/eth_account.go @@ -0,0 +1,175 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipld + +import ( + "encoding/json" + "fmt" + "math/big" + + "github.com/ipfs/go-cid" + node "github.com/ipfs/go-ipld-format" +) + +// EthAccountSnapshot (eth-account-snapshot codec 0x97) +// represents an ethereum account, i.e. a wallet address or +// a smart contract +type EthAccountSnapshot struct { + *EthAccount + + cid cid.Cid + rawdata []byte +} + +// EthAccount is the building block of EthAccountSnapshot. +// Or, is the former stripped of its cid and rawdata components. +type EthAccount struct { + Nonce uint64 + Balance *big.Int + Root []byte // This is the storage root trie + CodeHash []byte // This is the hash of the EVM code +} + +// Static (compile time) check that EthAccountSnapshot satisfies the +// node.Node interface. +var _ node.Node = (*EthAccountSnapshot)(nil) + +/* + INPUT +*/ + +// Input should be managed by EthStateTrie + +/* + OUTPUT +*/ + +// Output should be managed by EthStateTrie + +/* + Block INTERFACE +*/ + +// RawData returns the binary of the RLP encode of the account snapshot. +func (as *EthAccountSnapshot) RawData() []byte { + return as.rawdata +} + +// Cid returns the cid of the transaction. +func (as *EthAccountSnapshot) Cid() cid.Cid { + return as.cid +} + +// String is a helper for output +func (as *EthAccountSnapshot) String() string { + return fmt.Sprintf("", as.cid) +} + +// Loggable returns in a map the type of IPLD Link. +func (as *EthAccountSnapshot) Loggable() map[string]interface{} { + return map[string]interface{}{ + "type": "eth-account-snapshot", + } +} + +/* + Node INTERFACE +*/ + +// Resolve resolves a path through this node, stopping at any link boundary +// and returning the object found as well as the remaining path to traverse +func (as *EthAccountSnapshot) Resolve(p []string) (interface{}, []string, error) { + if len(p) == 0 { + return as, nil, nil + } + + if len(p) > 1 { + return nil, nil, fmt.Errorf("unexpected path elements past %s", p[0]) + } + + switch p[0] { + case "balance": + return as.Balance, nil, nil + case "codeHash": + return &node.Link{Cid: keccak256ToCid(RawBinary, as.CodeHash)}, nil, nil + case "nonce": + return as.Nonce, nil, nil + case "root": + return &node.Link{Cid: keccak256ToCid(MEthStorageTrie, as.Root)}, nil, nil + default: + return nil, nil, fmt.Errorf("no such link") + } +} + +// Tree lists all paths within the object under 'path', and up to the given depth. +// To list the entire object (similar to `find .`) pass "" and -1 +func (as *EthAccountSnapshot) Tree(p string, depth int) []string { + if p != "" || depth == 0 { + return nil + } + return []string{"balance", "codeHash", "nonce", "root"} +} + +// ResolveLink is a helper function that calls resolve and asserts the +// output is a link +func (as *EthAccountSnapshot) ResolveLink(p []string) (*node.Link, []string, error) { + obj, rest, err := as.Resolve(p) + if err != nil { + return nil, nil, err + } + + if lnk, ok := obj.(*node.Link); ok { + return lnk, rest, nil + } + + return nil, nil, fmt.Errorf("resolved item was not a link") +} + +// Copy will go away. It is here to comply with the interface. +func (as *EthAccountSnapshot) Copy() node.Node { + panic("dont use this yet") +} + +// Links is a helper function that returns all links within this object +func (as *EthAccountSnapshot) Links() []*node.Link { + return nil +} + +// Stat will go away. It is here to comply with the interface. +func (as *EthAccountSnapshot) Stat() (*node.NodeStat, error) { + return &node.NodeStat{}, nil +} + +// Size will go away. It is here to comply with the interface. +func (as *EthAccountSnapshot) Size() (uint64, error) { + return 0, nil +} + +/* + EthAccountSnapshot functions +*/ + +// MarshalJSON processes the transaction into readable JSON format. +func (as *EthAccountSnapshot) MarshalJSON() ([]byte, error) { + out := map[string]interface{}{ + "balance": as.Balance, + "codeHash": keccak256ToCid(RawBinary, as.CodeHash), + "nonce": as.Nonce, + "root": keccak256ToCid(MEthStorageTrie, as.Root), + } + return json.Marshal(out) +} diff --git a/statediff/indexer/ipfs/ipld/eth_header.go b/statediff/indexer/ipfs/ipld/eth_header.go new file mode 100644 index 000000000..c33931d4f --- /dev/null +++ b/statediff/indexer/ipfs/ipld/eth_header.go @@ -0,0 +1,256 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipld + +import ( + "encoding/json" + "fmt" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ipfs/go-cid" + node "github.com/ipfs/go-ipld-format" + mh "github.com/multiformats/go-multihash" +) + +// EthHeader (eth-block, codec 0x90), represents an ethereum block header +type EthHeader struct { + *types.Header + + cid cid.Cid + rawdata []byte +} + +// Static (compile time) check that EthHeader satisfies the node.Node interface. +var _ node.Node = (*EthHeader)(nil) + +/* + INPUT +*/ + +// NewEthHeader converts a *types.Header into an EthHeader IPLD node +func NewEthHeader(header *types.Header) (*EthHeader, error) { + headerRLP, err := rlp.EncodeToBytes(header) + if err != nil { + return nil, err + } + c, err := RawdataToCid(MEthHeader, headerRLP, mh.KECCAK_256) + if err != nil { + return nil, err + } + return &EthHeader{ + Header: header, + cid: c, + rawdata: headerRLP, + }, nil +} + +/* + OUTPUT +*/ + +// DecodeEthHeader takes a cid and its raw binary data +// from IPFS and returns an EthTx object for further processing. +func DecodeEthHeader(c cid.Cid, b []byte) (*EthHeader, error) { + var h *types.Header + if err := rlp.DecodeBytes(b, h); err != nil { + return nil, err + } + return &EthHeader{ + Header: h, + cid: c, + rawdata: b, + }, nil +} + +/* + Block INTERFACE +*/ + +// RawData returns the binary of the RLP encode of the block header. +func (b *EthHeader) RawData() []byte { + return b.rawdata +} + +// Cid returns the cid of the block header. +func (b *EthHeader) Cid() cid.Cid { + return b.cid +} + +// String is a helper for output +func (b *EthHeader) String() string { + return fmt.Sprintf("", b.cid) +} + +// Loggable returns a map the type of IPLD Link. +func (b *EthHeader) Loggable() map[string]interface{} { + return map[string]interface{}{ + "type": "eth-block", + } +} + +/* + Node INTERFACE +*/ + +// Resolve resolves a path through this node, stopping at any link boundary +// and returning the object found as well as the remaining path to traverse +func (b *EthHeader) Resolve(p []string) (interface{}, []string, error) { + if len(p) == 0 { + return b, nil, nil + } + + first, rest := p[0], p[1:] + + switch first { + case "parent": + return &node.Link{Cid: commonHashToCid(MEthHeader, b.ParentHash)}, rest, nil + case "receipts": + return &node.Link{Cid: commonHashToCid(MEthTxReceiptTrie, b.ReceiptHash)}, rest, nil + case "root": + return &node.Link{Cid: commonHashToCid(MEthStateTrie, b.Root)}, rest, nil + case "tx": + return &node.Link{Cid: commonHashToCid(MEthTxTrie, b.TxHash)}, rest, nil + case "uncles": + return &node.Link{Cid: commonHashToCid(MEthHeaderList, b.UncleHash)}, rest, nil + } + + if len(p) != 1 { + return nil, nil, fmt.Errorf("unexpected path elements past %s", first) + } + + switch first { + case "bloom": + return b.Bloom, nil, nil + case "coinbase": + return b.Coinbase, nil, nil + case "difficulty": + return b.Difficulty, nil, nil + case "extra": + // This is a []byte. By default they are marshalled into Base64. + return fmt.Sprintf("0x%x", b.Extra), nil, nil + case "gaslimit": + return b.GasLimit, nil, nil + case "gasused": + return b.GasUsed, nil, nil + case "mixdigest": + return b.MixDigest, nil, nil + case "nonce": + return b.Nonce, nil, nil + case "number": + return b.Number, nil, nil + case "time": + return b.Time, nil, nil + default: + return nil, nil, fmt.Errorf("no such link") + } +} + +// Tree lists all paths within the object under 'path', and up to the given depth. +// To list the entire object (similar to `find .`) pass "" and -1 +func (b *EthHeader) Tree(p string, depth int) []string { + if p != "" || depth == 0 { + return nil + } + + return []string{ + "time", + "bloom", + "coinbase", + "difficulty", + "extra", + "gaslimit", + "gasused", + "mixdigest", + "nonce", + "number", + "parent", + "receipts", + "root", + "tx", + "uncles", + } +} + +// ResolveLink is a helper function that allows easier traversal of links through blocks +func (b *EthHeader) ResolveLink(p []string) (*node.Link, []string, error) { + obj, rest, err := b.Resolve(p) + if err != nil { + return nil, nil, err + } + + if lnk, ok := obj.(*node.Link); ok { + return lnk, rest, nil + } + + return nil, nil, fmt.Errorf("resolved item was not a link") +} + +// Copy will go away. It is here to comply with the Node interface. +func (b *EthHeader) Copy() node.Node { + panic("implement me") +} + +// Links is a helper function that returns all links within this object +// HINT: Use `ipfs refs ` +func (b *EthHeader) Links() []*node.Link { + return []*node.Link{ + {Cid: commonHashToCid(MEthHeader, b.ParentHash)}, + {Cid: commonHashToCid(MEthTxReceiptTrie, b.ReceiptHash)}, + {Cid: commonHashToCid(MEthStateTrie, b.Root)}, + {Cid: commonHashToCid(MEthTxTrie, b.TxHash)}, + {Cid: commonHashToCid(MEthHeaderList, b.UncleHash)}, + } +} + +// Stat will go away. It is here to comply with the Node interface. +func (b *EthHeader) Stat() (*node.NodeStat, error) { + return &node.NodeStat{}, nil +} + +// Size will go away. It is here to comply with the Node interface. +func (b *EthHeader) Size() (uint64, error) { + return 0, nil +} + +/* + EthHeader functions +*/ + +// MarshalJSON processes the block header into readable JSON format, +// converting the right links into their cids, and keeping the original +// hex hash, allowing the user to simplify external queries. +func (b *EthHeader) MarshalJSON() ([]byte, error) { + out := map[string]interface{}{ + "time": b.Time, + "bloom": b.Bloom, + "coinbase": b.Coinbase, + "difficulty": b.Difficulty, + "extra": fmt.Sprintf("0x%x", b.Extra), + "gaslimit": b.GasLimit, + "gasused": b.GasUsed, + "mixdigest": b.MixDigest, + "nonce": b.Nonce, + "number": b.Number, + "parent": commonHashToCid(MEthHeader, b.ParentHash), + "receipts": commonHashToCid(MEthTxReceiptTrie, b.ReceiptHash), + "root": commonHashToCid(MEthStateTrie, b.Root), + "tx": commonHashToCid(MEthTxTrie, b.TxHash), + "uncles": commonHashToCid(MEthHeaderList, b.UncleHash), + } + return json.Marshal(out) +} diff --git a/statediff/indexer/ipfs/ipld/eth_parser.go b/statediff/indexer/ipfs/ipld/eth_parser.go new file mode 100644 index 000000000..f02d7d401 --- /dev/null +++ b/statediff/indexer/ipfs/ipld/eth_parser.go @@ -0,0 +1,97 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipld + +import ( + "bytes" + "fmt" + + "github.com/ethereum/go-ethereum/core/types" +) + +// FromBlockAndReceipts takes a block and processes it +// to return it a set of IPLD nodes for further processing. +func FromBlockAndReceipts(block *types.Block, receipts []*types.Receipt) (*EthHeader, []*EthHeader, []*EthTx, []*EthTxTrie, []*EthReceipt, []*EthRctTrie, error) { + // Process the header + headerNode, err := NewEthHeader(block.Header()) + if err != nil { + return nil, nil, nil, nil, nil, nil, err + } + // Process the uncles + uncleNodes := make([]*EthHeader, len(block.Uncles())) + for i, uncle := range block.Uncles() { + uncleNode, err := NewEthHeader(uncle) + if err != nil { + return nil, nil, nil, nil, nil, nil, err + } + uncleNodes[i] = uncleNode + } + // Process the txs + ethTxNodes, ethTxTrieNodes, err := processTransactions(block.Transactions(), + block.Header().TxHash[:]) + if err != nil { + return nil, nil, nil, nil, nil, nil, err + } + // Process the receipts + ethRctNodes, ethRctTrieNodes, err := processReceipts(receipts, + block.Header().ReceiptHash[:]) + return headerNode, uncleNodes, ethTxNodes, ethTxTrieNodes, ethRctNodes, ethRctTrieNodes, err +} + +// processTransactions will take the found transactions in a parsed block body +// to return IPLD node slices for eth-tx and eth-tx-trie +func processTransactions(txs []*types.Transaction, expectedTxRoot []byte) ([]*EthTx, []*EthTxTrie, error) { + var ethTxNodes []*EthTx + transactionTrie := newTxTrie() + + for idx, tx := range txs { + ethTx, err := NewEthTx(tx) + if err != nil { + return nil, nil, err + } + ethTxNodes = append(ethTxNodes, ethTx) + transactionTrie.add(idx, ethTx.RawData()) + } + + if !bytes.Equal(transactionTrie.rootHash(), expectedTxRoot) { + return nil, nil, fmt.Errorf("wrong transaction hash computed") + } + + return ethTxNodes, transactionTrie.getNodes(), nil +} + +// processReceipts will take in receipts +// to return IPLD node slices for eth-rct and eth-rct-trie +func processReceipts(rcts []*types.Receipt, expectedRctRoot []byte) ([]*EthReceipt, []*EthRctTrie, error) { + var ethRctNodes []*EthReceipt + receiptTrie := newRctTrie() + + for idx, rct := range rcts { + ethRct, err := NewReceipt(rct) + if err != nil { + return nil, nil, err + } + ethRctNodes = append(ethRctNodes, ethRct) + receiptTrie.add(idx, ethRct.RawData()) + } + + if !bytes.Equal(receiptTrie.rootHash(), expectedRctRoot) { + return nil, nil, fmt.Errorf("wrong receipt hash computed") + } + + return ethRctNodes, receiptTrie.getNodes(), nil +} diff --git a/statediff/indexer/ipfs/ipld/eth_receipt.go b/statediff/indexer/ipfs/ipld/eth_receipt.go new file mode 100644 index 000000000..cfa46b36e --- /dev/null +++ b/statediff/indexer/ipfs/ipld/eth_receipt.go @@ -0,0 +1,199 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipld + +import ( + "encoding/json" + "fmt" + "strconv" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ipfs/go-cid" + node "github.com/ipfs/go-ipld-format" + mh "github.com/multiformats/go-multihash" +) + +type EthReceipt struct { + *types.Receipt + + rawdata []byte + cid cid.Cid +} + +// Static (compile time) check that EthReceipt satisfies the node.Node interface. +var _ node.Node = (*EthReceipt)(nil) + +/* + INPUT +*/ + +// NewReceipt converts a types.ReceiptForStorage to an EthReceipt IPLD node +func NewReceipt(receipt *types.Receipt) (*EthReceipt, error) { + receiptRLP, err := rlp.EncodeToBytes(receipt) + if err != nil { + return nil, err + } + c, err := RawdataToCid(MEthTxReceipt, receiptRLP, mh.KECCAK_256) + if err != nil { + return nil, err + } + return &EthReceipt{ + Receipt: receipt, + cid: c, + rawdata: receiptRLP, + }, nil +} + +/* + OUTPUT +*/ + +// DecodeEthReceipt takes a cid and its raw binary data +// from IPFS and returns an EthTx object for further processing. +func DecodeEthReceipt(c cid.Cid, b []byte) (*EthReceipt, error) { + var r *types.Receipt + if err := rlp.DecodeBytes(b, r); err != nil { + return nil, err + } + return &EthReceipt{ + Receipt: r, + cid: c, + rawdata: b, + }, nil +} + +/* + Block INTERFACE +*/ + +func (node *EthReceipt) RawData() []byte { + return node.rawdata +} + +func (node *EthReceipt) Cid() cid.Cid { + return node.cid +} + +// String is a helper for output +func (r *EthReceipt) String() string { + return fmt.Sprintf("", r.cid) +} + +// Loggable returns in a map the type of IPLD Link. +func (r *EthReceipt) Loggable() map[string]interface{} { + return map[string]interface{}{ + "type": "eth-receipt", + } +} + +// Resolve resolves a path through this node, stopping at any link boundary +// and returning the object found as well as the remaining path to traverse +func (r *EthReceipt) Resolve(p []string) (interface{}, []string, error) { + if len(p) == 0 { + return r, nil, nil + } + + if len(p) > 1 { + return nil, nil, fmt.Errorf("unexpected path elements past %s", p[0]) + } + + switch p[0] { + + case "root": + return r.PostState, nil, nil + case "status": + return r.Status, nil, nil + case "cumulativeGasUsed": + return r.CumulativeGasUsed, nil, nil + case "logsBloom": + return r.Bloom, nil, nil + case "logs": + return r.Logs, nil, nil + case "transactionHash": + return r.TxHash, nil, nil + case "contractAddress": + return r.ContractAddress, nil, nil + case "gasUsed": + return r.GasUsed, nil, nil + default: + return nil, nil, fmt.Errorf("no such link") + } +} + +// Tree lists all paths within the object under 'path', and up to the given depth. +// To list the entire object (similar to `find .`) pass "" and -1 +func (r *EthReceipt) Tree(p string, depth int) []string { + if p != "" || depth == 0 { + return nil + } + return []string{"root", "status", "cumulativeGasUsed", "logsBloom", "logs", "transactionHash", "contractAddress", "gasUsed"} +} + +// ResolveLink is a helper function that calls resolve and asserts the +// output is a link +func (r *EthReceipt) ResolveLink(p []string) (*node.Link, []string, error) { + obj, rest, err := r.Resolve(p) + if err != nil { + return nil, nil, err + } + + if lnk, ok := obj.(*node.Link); ok { + return lnk, rest, nil + } + + return nil, nil, fmt.Errorf("resolved item was not a link") +} + +// Copy will go away. It is here to comply with the Node interface. +func (*EthReceipt) Copy() node.Node { + panic("implement me") +} + +// Links is a helper function that returns all links within this object +func (*EthReceipt) Links() []*node.Link { + return nil +} + +// Stat will go away. It is here to comply with the interface. +func (r *EthReceipt) Stat() (*node.NodeStat, error) { + return &node.NodeStat{}, nil +} + +// Size will go away. It is here to comply with the interface. +func (r *EthReceipt) Size() (uint64, error) { + return strconv.ParseUint(r.Receipt.Size().String(), 10, 64) +} + +/* + EthReceipt functions +*/ + +// MarshalJSON processes the receipt into readable JSON format. +func (r *EthReceipt) MarshalJSON() ([]byte, error) { + out := map[string]interface{}{ + "root": r.PostState, + "status": r.Status, + "cumulativeGasUsed": r.CumulativeGasUsed, + "logsBloom": r.Bloom, + "logs": r.Logs, + "transactionHash": r.TxHash, + "contractAddress": r.ContractAddress, + "gasUsed": r.GasUsed, + } + return json.Marshal(out) +} diff --git a/statediff/indexer/ipfs/ipld/eth_receipt_trie.go b/statediff/indexer/ipfs/ipld/eth_receipt_trie.go new file mode 100644 index 000000000..6a1b7e406 --- /dev/null +++ b/statediff/indexer/ipfs/ipld/eth_receipt_trie.go @@ -0,0 +1,152 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipld + +import ( + "fmt" + + "github.com/ipfs/go-cid" + node "github.com/ipfs/go-ipld-format" + "github.com/multiformats/go-multihash" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" +) + +// EthRctTrie (eth-tx-trie codec 0x92) represents +// a node from the transaction trie in ethereum. +type EthRctTrie struct { + *TrieNode +} + +// Static (compile time) check that EthRctTrie satisfies the node.Node interface. +var _ node.Node = (*EthRctTrie)(nil) + +/* + INPUT +*/ + +// To create a proper trie of the eth-tx-trie objects, it is required +// to input all transactions belonging to a forest in a single step. +// We are adding the transactions, and creating its trie on +// block body parsing time. + +/* + OUTPUT +*/ + +// DecodeEthRctTrie returns an EthRctTrie object from its cid and rawdata. +func DecodeEthRctTrie(c cid.Cid, b []byte) (*EthRctTrie, error) { + tn, err := decodeTrieNode(c, b, decodeEthRctTrieLeaf) + if err != nil { + return nil, err + } + return &EthRctTrie{TrieNode: tn}, nil +} + +// decodeEthRctTrieLeaf parses a eth-rct-trie leaf +//from decoded RLP elements +func decodeEthRctTrieLeaf(i []interface{}) ([]interface{}, error) { + var r types.Receipt + err := rlp.DecodeBytes(i[1].([]byte), &r) + if err != nil { + return nil, err + } + c, err := RawdataToCid(MEthTxReceipt, i[1].([]byte), multihash.KECCAK_256) + if err != nil { + return nil, err + } + return []interface{}{ + i[0].([]byte), + &EthReceipt{ + Receipt: &r, + cid: c, + rawdata: i[1].([]byte), + }, + }, nil +} + +/* + Block INTERFACE +*/ + +// RawData returns the binary of the RLP encode of the transaction. +func (t *EthRctTrie) RawData() []byte { + return t.rawdata +} + +// Cid returns the cid of the transaction. +func (t *EthRctTrie) Cid() cid.Cid { + return t.cid +} + +// String is a helper for output +func (t *EthRctTrie) String() string { + return fmt.Sprintf("", t.cid) +} + +// Loggable returns in a map the type of IPLD Link. +func (t *EthRctTrie) Loggable() map[string]interface{} { + return map[string]interface{}{ + "type": "eth-rct-trie", + } +} + +/* + EthRctTrie functions +*/ + +// rctTrie wraps a localTrie for use on the receipt trie. +type rctTrie struct { + *localTrie +} + +// newRctTrie initializes and returns a rctTrie. +func newRctTrie() *rctTrie { + return &rctTrie{ + localTrie: newLocalTrie(), + } +} + +// getNodes invokes the localTrie, which computes the root hash of the +// transaction trie and returns its database keys, to return a slice +// of EthRctTrie nodes. +func (rt *rctTrie) getNodes() []*EthRctTrie { + keys := rt.getKeys() + var out []*EthRctTrie + it := rt.trie.NodeIterator([]byte{}) + for it.Next(true) { + + } + for _, k := range keys { + rawdata, err := rt.db.Get(k) + if err != nil { + panic(err) + } + c, err := RawdataToCid(MEthTxReceiptTrie, rawdata, multihash.KECCAK_256) + if err != nil { + return nil + } + tn := &TrieNode{ + cid: c, + rawdata: rawdata, + } + out = append(out, &EthRctTrie{TrieNode: tn}) + } + + return out +} diff --git a/statediff/indexer/ipfs/ipld/eth_state.go b/statediff/indexer/ipfs/ipld/eth_state.go new file mode 100644 index 000000000..a127f956e --- /dev/null +++ b/statediff/indexer/ipfs/ipld/eth_state.go @@ -0,0 +1,114 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipld + +import ( + "fmt" + + "github.com/ipfs/go-cid" + node "github.com/ipfs/go-ipld-format" + "github.com/multiformats/go-multihash" + + "github.com/ethereum/go-ethereum/rlp" +) + +// EthStateTrie (eth-state-trie, codec 0x96), represents +// a node from the satte trie in ethereum. +type EthStateTrie struct { + *TrieNode +} + +// Static (compile time) check that EthStateTrie satisfies the node.Node interface. +var _ node.Node = (*EthStateTrie)(nil) + +/* + INPUT +*/ + +// FromStateTrieRLP takes the RLP representation of an ethereum +// state trie node to return it as an IPLD node for further processing. +func FromStateTrieRLP(raw []byte) (*EthStateTrie, error) { + c, err := RawdataToCid(MEthStateTrie, raw, multihash.KECCAK_256) + if err != nil { + return nil, err + } + // Let's run the whole mile and process the nodeKind and + // its elements, in case somebody would need this function + // to parse an RLP element from the filesystem + return DecodeEthStateTrie(c, raw) +} + +/* + OUTPUT +*/ + +// DecodeEthStateTrie returns an EthStateTrie object from its cid and rawdata. +func DecodeEthStateTrie(c cid.Cid, b []byte) (*EthStateTrie, error) { + tn, err := decodeTrieNode(c, b, decodeEthStateTrieLeaf) + if err != nil { + return nil, err + } + return &EthStateTrie{TrieNode: tn}, nil +} + +// decodeEthStateTrieLeaf parses a eth-tx-trie leaf +// from decoded RLP elements +func decodeEthStateTrieLeaf(i []interface{}) ([]interface{}, error) { + var account EthAccount + err := rlp.DecodeBytes(i[1].([]byte), &account) + if err != nil { + return nil, err + } + c, err := RawdataToCid(MEthAccountSnapshot, i[1].([]byte), multihash.KECCAK_256) + if err != nil { + return nil, err + } + return []interface{}{ + i[0].([]byte), + &EthAccountSnapshot{ + EthAccount: &account, + cid: c, + rawdata: i[1].([]byte), + }, + }, nil +} + +/* + Block INTERFACE +*/ + +// RawData returns the binary of the RLP encode of the state trie node. +func (st *EthStateTrie) RawData() []byte { + return st.rawdata +} + +// Cid returns the cid of the state trie node. +func (st *EthStateTrie) Cid() cid.Cid { + return st.cid +} + +// String is a helper for output +func (st *EthStateTrie) String() string { + return fmt.Sprintf("", st.cid) +} + +// Loggable returns in a map the type of IPLD Link. +func (st *EthStateTrie) Loggable() map[string]interface{} { + return map[string]interface{}{ + "type": "eth-state-trie", + } +} diff --git a/statediff/indexer/ipfs/ipld/eth_storage.go b/statediff/indexer/ipfs/ipld/eth_storage.go new file mode 100644 index 000000000..779cad4d9 --- /dev/null +++ b/statediff/indexer/ipfs/ipld/eth_storage.go @@ -0,0 +1,100 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipld + +import ( + "fmt" + + "github.com/ipfs/go-cid" + node "github.com/ipfs/go-ipld-format" + "github.com/multiformats/go-multihash" +) + +// EthStorageTrie (eth-storage-trie, codec 0x98), represents +// a node from the storage trie in ethereum. +type EthStorageTrie struct { + *TrieNode +} + +// Static (compile time) check that EthStorageTrie satisfies the node.Node interface. +var _ node.Node = (*EthStorageTrie)(nil) + +/* + INPUT +*/ + +// FromStorageTrieRLP takes the RLP representation of an ethereum +// storage trie node to return it as an IPLD node for further processing. +func FromStorageTrieRLP(raw []byte) (*EthStorageTrie, error) { + c, err := RawdataToCid(MEthStorageTrie, raw, multihash.KECCAK_256) + if err != nil { + return nil, err + } + + // Let's run the whole mile and process the nodeKind and + // its elements, in case somebody would need this function + // to parse an RLP element from the filesystem + return DecodeEthStorageTrie(c, raw) +} + +/* + OUTPUT +*/ + +// DecodeEthStorageTrie returns an EthStorageTrie object from its cid and rawdata. +func DecodeEthStorageTrie(c cid.Cid, b []byte) (*EthStorageTrie, error) { + tn, err := decodeTrieNode(c, b, decodeEthStorageTrieLeaf) + if err != nil { + return nil, err + } + return &EthStorageTrie{TrieNode: tn}, nil +} + +// decodeEthStorageTrieLeaf parses a eth-tx-trie leaf +// from decoded RLP elements +func decodeEthStorageTrieLeaf(i []interface{}) ([]interface{}, error) { + return []interface{}{ + i[0].([]byte), + i[1].([]byte), + }, nil +} + +/* + Block INTERFACE +*/ + +// RawData returns the binary of the RLP encode of the storage trie node. +func (st *EthStorageTrie) RawData() []byte { + return st.rawdata +} + +// Cid returns the cid of the storage trie node. +func (st *EthStorageTrie) Cid() cid.Cid { + return st.cid +} + +// String is a helper for output +func (st *EthStorageTrie) String() string { + return fmt.Sprintf("", st.cid) +} + +// Loggable returns in a map the type of IPLD Link. +func (st *EthStorageTrie) Loggable() map[string]interface{} { + return map[string]interface{}{ + "type": "eth-storage-trie", + } +} diff --git a/statediff/indexer/ipfs/ipld/eth_tx.go b/statediff/indexer/ipfs/ipld/eth_tx.go new file mode 100644 index 000000000..4fc4d20a6 --- /dev/null +++ b/statediff/indexer/ipfs/ipld/eth_tx.go @@ -0,0 +1,215 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipld + +import ( + "encoding/json" + "fmt" + "strconv" + + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ipfs/go-cid" + node "github.com/ipfs/go-ipld-format" + mh "github.com/multiformats/go-multihash" +) + +// EthTx (eth-tx codec 0x93) represents an ethereum transaction +type EthTx struct { + *types.Transaction + + cid cid.Cid + rawdata []byte +} + +// Static (compile time) check that EthTx satisfies the node.Node interface. +var _ node.Node = (*EthTx)(nil) + +/* + INPUT +*/ + +// NewEthTx converts a *types.Transaction to an EthTx IPLD node +func NewEthTx(tx *types.Transaction) (*EthTx, error) { + txRLP, err := rlp.EncodeToBytes(tx) + if err != nil { + return nil, err + } + c, err := RawdataToCid(MEthTx, txRLP, mh.KECCAK_256) + if err != nil { + return nil, err + } + return &EthTx{ + Transaction: tx, + cid: c, + rawdata: txRLP, + }, nil +} + +/* + OUTPUT +*/ + +// DecodeEthTx takes a cid and its raw binary data +// from IPFS and returns an EthTx object for further processing. +func DecodeEthTx(c cid.Cid, b []byte) (*EthTx, error) { + var t *types.Transaction + if err := rlp.DecodeBytes(b, t); err != nil { + return nil, err + } + return &EthTx{ + Transaction: t, + cid: c, + rawdata: b, + }, nil +} + +/* + Block INTERFACE +*/ + +// RawData returns the binary of the RLP encode of the transaction. +func (t *EthTx) RawData() []byte { + return t.rawdata +} + +// Cid returns the cid of the transaction. +func (t *EthTx) Cid() cid.Cid { + return t.cid +} + +// String is a helper for output +func (t *EthTx) String() string { + return fmt.Sprintf("", t.cid) +} + +// Loggable returns in a map the type of IPLD Link. +func (t *EthTx) Loggable() map[string]interface{} { + return map[string]interface{}{ + "type": "eth-tx", + } +} + +/* + Node INTERFACE +*/ + +// Resolve resolves a path through this node, stopping at any link boundary +// and returning the object found as well as the remaining path to traverse +func (t *EthTx) Resolve(p []string) (interface{}, []string, error) { + if len(p) == 0 { + return t, nil, nil + } + + if len(p) > 1 { + return nil, nil, fmt.Errorf("unexpected path elements past %s", p[0]) + } + + switch p[0] { + + case "gas": + return t.Gas(), nil, nil + case "gasPrice": + return t.GasPrice(), nil, nil + case "input": + return fmt.Sprintf("%x", t.Data()), nil, nil + case "nonce": + return t.Nonce(), nil, nil + case "r": + _, r, _ := t.RawSignatureValues() + return hexutil.EncodeBig(r), nil, nil + case "s": + _, _, s := t.RawSignatureValues() + return hexutil.EncodeBig(s), nil, nil + case "toAddress": + return t.To(), nil, nil + case "v": + v, _, _ := t.RawSignatureValues() + return hexutil.EncodeBig(v), nil, nil + case "value": + return hexutil.EncodeBig(t.Value()), nil, nil + default: + return nil, nil, fmt.Errorf("no such link") + } +} + +// Tree lists all paths within the object under 'path', and up to the given depth. +// To list the entire object (similar to `find .`) pass "" and -1 +func (t *EthTx) Tree(p string, depth int) []string { + if p != "" || depth == 0 { + return nil + } + return []string{"gas", "gasPrice", "input", "nonce", "r", "s", "toAddress", "v", "value"} +} + +// ResolveLink is a helper function that calls resolve and asserts the +// output is a link +func (t *EthTx) ResolveLink(p []string) (*node.Link, []string, error) { + obj, rest, err := t.Resolve(p) + if err != nil { + return nil, nil, err + } + + if lnk, ok := obj.(*node.Link); ok { + return lnk, rest, nil + } + + return nil, nil, fmt.Errorf("resolved item was not a link") +} + +// Copy will go away. It is here to comply with the interface. +func (t *EthTx) Copy() node.Node { + panic("implement me") +} + +// Links is a helper function that returns all links within this object +func (t *EthTx) Links() []*node.Link { + return nil +} + +// Stat will go away. It is here to comply with the interface. +func (t *EthTx) Stat() (*node.NodeStat, error) { + return &node.NodeStat{}, nil +} + +// Size will go away. It is here to comply with the interface. +func (t *EthTx) Size() (uint64, error) { + return strconv.ParseUint(t.Transaction.Size().String(), 10, 64) +} + +/* + EthTx functions +*/ + +// MarshalJSON processes the transaction into readable JSON format. +func (t *EthTx) MarshalJSON() ([]byte, error) { + v, r, s := t.RawSignatureValues() + + out := map[string]interface{}{ + "gas": t.Gas(), + "gasPrice": hexutil.EncodeBig(t.GasPrice()), + "input": fmt.Sprintf("%x", t.Data()), + "nonce": t.Nonce(), + "r": hexutil.EncodeBig(r), + "s": hexutil.EncodeBig(s), + "toAddress": t.To(), + "v": hexutil.EncodeBig(v), + "value": hexutil.EncodeBig(t.Value()), + } + return json.Marshal(out) +} diff --git a/statediff/indexer/ipfs/ipld/eth_tx_trie.go b/statediff/indexer/ipfs/ipld/eth_tx_trie.go new file mode 100644 index 000000000..6f106f6d8 --- /dev/null +++ b/statediff/indexer/ipfs/ipld/eth_tx_trie.go @@ -0,0 +1,152 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipld + +import ( + "fmt" + + "github.com/ipfs/go-cid" + node "github.com/ipfs/go-ipld-format" + "github.com/multiformats/go-multihash" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" +) + +// EthTxTrie (eth-tx-trie codec 0x92) represents +// a node from the transaction trie in ethereum. +type EthTxTrie struct { + *TrieNode +} + +// Static (compile time) check that EthTxTrie satisfies the node.Node interface. +var _ node.Node = (*EthTxTrie)(nil) + +/* + INPUT +*/ + +// To create a proper trie of the eth-tx-trie objects, it is required +// to input all transactions belonging to a forest in a single step. +// We are adding the transactions, and creating its trie on +// block body parsing time. + +/* + OUTPUT +*/ + +// DecodeEthTxTrie returns an EthTxTrie object from its cid and rawdata. +func DecodeEthTxTrie(c cid.Cid, b []byte) (*EthTxTrie, error) { + tn, err := decodeTrieNode(c, b, decodeEthTxTrieLeaf) + if err != nil { + return nil, err + } + return &EthTxTrie{TrieNode: tn}, nil +} + +// decodeEthTxTrieLeaf parses a eth-tx-trie leaf +//from decoded RLP elements +func decodeEthTxTrieLeaf(i []interface{}) ([]interface{}, error) { + var t types.Transaction + err := rlp.DecodeBytes(i[1].([]byte), &t) + if err != nil { + return nil, err + } + c, err := RawdataToCid(MEthTx, i[1].([]byte), multihash.KECCAK_256) + if err != nil { + return nil, err + } + return []interface{}{ + i[0].([]byte), + &EthTx{ + Transaction: &t, + cid: c, + rawdata: i[1].([]byte), + }, + }, nil +} + +/* + Block INTERFACE +*/ + +// RawData returns the binary of the RLP encode of the transaction. +func (t *EthTxTrie) RawData() []byte { + return t.rawdata +} + +// Cid returns the cid of the transaction. +func (t *EthTxTrie) Cid() cid.Cid { + return t.cid +} + +// String is a helper for output +func (t *EthTxTrie) String() string { + return fmt.Sprintf("", t.cid) +} + +// Loggable returns in a map the type of IPLD Link. +func (t *EthTxTrie) Loggable() map[string]interface{} { + return map[string]interface{}{ + "type": "eth-tx-trie", + } +} + +/* + EthTxTrie functions +*/ + +// txTrie wraps a localTrie for use on the transaction trie. +type txTrie struct { + *localTrie +} + +// newTxTrie initializes and returns a txTrie. +func newTxTrie() *txTrie { + return &txTrie{ + localTrie: newLocalTrie(), + } +} + +// getNodes invokes the localTrie, which computes the root hash of the +// transaction trie and returns its database keys, to return a slice +// of EthTxTrie nodes. +func (tt *txTrie) getNodes() []*EthTxTrie { + keys := tt.getKeys() + var out []*EthTxTrie + it := tt.trie.NodeIterator([]byte{}) + for it.Next(true) { + + } + for _, k := range keys { + rawdata, err := tt.db.Get(k) + if err != nil { + panic(err) + } + c, err := RawdataToCid(MEthTxTrie, rawdata, multihash.KECCAK_256) + if err != nil { + return nil + } + tn := &TrieNode{ + cid: c, + rawdata: rawdata, + } + out = append(out, &EthTxTrie{TrieNode: tn}) + } + + return out +} diff --git a/statediff/indexer/ipfs/ipld/shared.go b/statediff/indexer/ipfs/ipld/shared.go new file mode 100644 index 000000000..cfdd7446a --- /dev/null +++ b/statediff/indexer/ipfs/ipld/shared.go @@ -0,0 +1,126 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipld + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" + "github.com/ipfs/go-cid" + mh "github.com/multiformats/go-multihash" +) + +// IPLD Codecs for Ethereum +// See the authoritative document: +// https://github.com/multiformats/multicodec/blob/master/table.csv +const ( + RawBinary = 0x55 + MEthHeader = 0x90 + MEthHeaderList = 0x91 + MEthTxTrie = 0x92 + MEthTx = 0x93 + MEthTxReceiptTrie = 0x94 + MEthTxReceipt = 0x95 + MEthStateTrie = 0x96 + MEthAccountSnapshot = 0x97 + MEthStorageTrie = 0x98 +) + +// RawdataToCid takes the desired codec and a slice of bytes +// and returns the proper cid of the object. +func RawdataToCid(codec uint64, rawdata []byte, multiHash uint64) (cid.Cid, error) { + c, err := cid.Prefix{ + Codec: codec, + Version: 1, + MhType: multiHash, + MhLength: -1, + }.Sum(rawdata) + if err != nil { + return cid.Cid{}, err + } + return c, nil +} + +// keccak256ToCid takes a keccak256 hash and returns its cid based on +// the codec given. +func keccak256ToCid(codec uint64, h []byte) cid.Cid { + buf, err := mh.Encode(h, mh.KECCAK_256) + if err != nil { + panic(err) + } + + return cid.NewCidV1(codec, mh.Multihash(buf)) +} + +// commonHashToCid takes a go-ethereum common.Hash and returns its +// cid based on the codec given, +func commonHashToCid(codec uint64, h common.Hash) cid.Cid { + mhash, err := mh.Encode(h[:], mh.KECCAK_256) + if err != nil { + panic(err) + } + + return cid.NewCidV1(codec, mhash) +} + +// localTrie wraps a go-ethereum trie and its underlying memory db. +// It contributes to the creation of the trie node objects. +type localTrie struct { + keys [][]byte + db ethdb.Database + trie *trie.Trie +} + +// newLocalTrie initializes and returns a localTrie object +func newLocalTrie() *localTrie { + var err error + lt := &localTrie{} + lt.db = rawdb.NewMemoryDatabase() + lt.trie, err = trie.New(common.Hash{}, trie.NewDatabase(lt.db)) + if err != nil { + panic(err) + } + return lt +} + +// add receives the index of an object and its rawdata value +// and includes it into the localTrie +func (lt *localTrie) add(idx int, rawdata []byte) { + key, err := rlp.EncodeToBytes(uint(idx)) + if err != nil { + panic(err) + } + lt.keys = append(lt.keys, key) + if err := lt.db.Put(key, rawdata); err != nil { + panic(err) + } + lt.trie.Update(key, rawdata) +} + +// rootHash returns the computed trie root. +// Useful for sanity checks on parsed data. +func (lt *localTrie) rootHash() []byte { + return lt.trie.Hash().Bytes() +} + +// getKeys returns the stored keys of the memory database +// of the localTrie for further processing. +func (lt *localTrie) getKeys() [][]byte { + return lt.keys +} diff --git a/statediff/indexer/ipfs/ipld/trie_node.go b/statediff/indexer/ipfs/ipld/trie_node.go new file mode 100644 index 000000000..788f76db8 --- /dev/null +++ b/statediff/indexer/ipfs/ipld/trie_node.go @@ -0,0 +1,444 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipld + +import ( + "encoding/json" + "fmt" + + "github.com/ethereum/go-ethereum/rlp" + "github.com/ipfs/go-cid" + node "github.com/ipfs/go-ipld-format" +) + +// TrieNode is the general abstraction for +//ethereum IPLD trie nodes. +type TrieNode struct { + // leaf, extension or branch + nodeKind string + + // If leaf or extension: [0] is key, [1] is val. + // If branch: [0] - [16] are children. + elements []interface{} + + // IPLD block information + cid cid.Cid + rawdata []byte +} + +/* + OUTPUT +*/ + +type trieNodeLeafDecoder func([]interface{}) ([]interface{}, error) + +// decodeTrieNode returns a TrieNode object from an IPLD block's +// cid and rawdata. +func decodeTrieNode(c cid.Cid, b []byte, + leafDecoder trieNodeLeafDecoder) (*TrieNode, error) { + var ( + i, decoded, elements []interface{} + nodeKind string + err error + ) + + if err = rlp.DecodeBytes(b, &i); err != nil { + return nil, err + } + + codec := c.Type() + switch len(i) { + case 2: + nodeKind, decoded, err = decodeCompactKey(i) + if err != nil { + return nil, err + } + + if nodeKind == "extension" { + elements, err = parseTrieNodeExtension(decoded, codec) + } + if nodeKind == "leaf" { + elements, err = leafDecoder(decoded) + } + if nodeKind != "extension" && nodeKind != "leaf" { + return nil, fmt.Errorf("unexpected nodeKind returned from decoder") + } + case 17: + nodeKind = "branch" + elements, err = parseTrieNodeBranch(i, codec) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("unknown trie node type") + } + + return &TrieNode{ + nodeKind: nodeKind, + elements: elements, + rawdata: b, + cid: c, + }, nil +} + +// decodeCompactKey takes a compact key, and returns its nodeKind and value. +func decodeCompactKey(i []interface{}) (string, []interface{}, error) { + first := i[0].([]byte) + last := i[1].([]byte) + + switch first[0] / 16 { + case '\x00': + return "extension", []interface{}{ + nibbleToByte(first)[2:], + last, + }, nil + case '\x01': + return "extension", []interface{}{ + nibbleToByte(first)[1:], + last, + }, nil + case '\x02': + return "leaf", []interface{}{ + nibbleToByte(first)[2:], + last, + }, nil + case '\x03': + return "leaf", []interface{}{ + nibbleToByte(first)[1:], + last, + }, nil + default: + return "", nil, fmt.Errorf("unknown hex prefix") + } +} + +// parseTrieNodeExtension helper improves readability +func parseTrieNodeExtension(i []interface{}, codec uint64) ([]interface{}, error) { + return []interface{}{ + i[0].([]byte), + keccak256ToCid(codec, i[1].([]byte)), + }, nil +} + +// parseTrieNodeBranch helper improves readability +func parseTrieNodeBranch(i []interface{}, codec uint64) ([]interface{}, error) { + var out []interface{} + + for i, vi := range i { + v, ok := vi.([]byte) + // Sometimes this throws "panic: interface conversion: interface {} is []interface {}, not []uint8" + // Figure out why, and if it is okay to continue + if !ok { + return nil, fmt.Errorf("unable to decode branch node entry into []byte at position: %d value: %+v", i, vi) + } + + switch len(v) { + case 0: + out = append(out, nil) + case 32: + out = append(out, keccak256ToCid(codec, v)) + default: + return nil, fmt.Errorf("unrecognized object: %v", v) + } + } + + return out, nil +} + +/* + Node INTERFACE +*/ + +// Resolve resolves a path through this node, stopping at any link boundary +// and returning the object found as well as the remaining path to traverse +func (t *TrieNode) Resolve(p []string) (interface{}, []string, error) { + switch t.nodeKind { + case "extension": + return t.resolveTrieNodeExtension(p) + case "leaf": + return t.resolveTrieNodeLeaf(p) + case "branch": + return t.resolveTrieNodeBranch(p) + default: + return nil, nil, fmt.Errorf("nodeKind case not implemented") + } +} + +// Tree lists all paths within the object under 'path', and up to the given depth. +// To list the entire object (similar to `find .`) pass "" and -1 +func (t *TrieNode) Tree(p string, depth int) []string { + if p != "" || depth == 0 { + return nil + } + + var out []string + + switch t.nodeKind { + case "extension": + var val string + for _, e := range t.elements[0].([]byte) { + val += fmt.Sprintf("%x", e) + } + return []string{val} + case "branch": + for i, elem := range t.elements { + if _, ok := elem.(*cid.Cid); ok { + out = append(out, fmt.Sprintf("%x", i)) + } + } + return out + + default: + return nil + } +} + +// ResolveLink is a helper function that calls resolve and asserts the +// output is a link +func (t *TrieNode) ResolveLink(p []string) (*node.Link, []string, error) { + obj, rest, err := t.Resolve(p) + if err != nil { + return nil, nil, err + } + + lnk, ok := obj.(*node.Link) + if !ok { + return nil, nil, fmt.Errorf("was not a link") + } + + return lnk, rest, nil +} + +// Copy will go away. It is here to comply with the interface. +func (t *TrieNode) Copy() node.Node { + panic("dont use this yet") +} + +// Links is a helper function that returns all links within this object +func (t *TrieNode) Links() []*node.Link { + var out []*node.Link + + for _, i := range t.elements { + c, ok := i.(cid.Cid) + if ok { + out = append(out, &node.Link{Cid: c}) + } + } + + return out +} + +// Stat will go away. It is here to comply with the interface. +func (t *TrieNode) Stat() (*node.NodeStat, error) { + return &node.NodeStat{}, nil +} + +// Size will go away. It is here to comply with the interface. +func (t *TrieNode) Size() (uint64, error) { + return 0, nil +} + +/* + TrieNode functions +*/ + +// MarshalJSON processes the transaction trie into readable JSON format. +func (t *TrieNode) MarshalJSON() ([]byte, error) { + var out map[string]interface{} + + switch t.nodeKind { + case "extension": + fallthrough + case "leaf": + var hexPrefix string + for _, e := range t.elements[0].([]byte) { + hexPrefix += fmt.Sprintf("%x", e) + } + + // if we got a byte we need to do this casting otherwise + // it will be marshaled to a base64 encoded value + if _, ok := t.elements[1].([]byte); ok { + var hexVal string + for _, e := range t.elements[1].([]byte) { + hexVal += fmt.Sprintf("%x", e) + } + + t.elements[1] = hexVal + } + + out = map[string]interface{}{ + "type": t.nodeKind, + hexPrefix: t.elements[1], + } + + case "branch": + out = map[string]interface{}{ + "type": "branch", + "0": t.elements[0], + "1": t.elements[1], + "2": t.elements[2], + "3": t.elements[3], + "4": t.elements[4], + "5": t.elements[5], + "6": t.elements[6], + "7": t.elements[7], + "8": t.elements[8], + "9": t.elements[9], + "a": t.elements[10], + "b": t.elements[11], + "c": t.elements[12], + "d": t.elements[13], + "e": t.elements[14], + "f": t.elements[15], + } + default: + return nil, fmt.Errorf("nodeKind %s not supported", t.nodeKind) + } + + return json.Marshal(out) +} + +// nibbleToByte expands the nibbles of a byte slice into their own bytes. +func nibbleToByte(k []byte) []byte { + var out []byte + + for _, b := range k { + out = append(out, b/16) + out = append(out, b%16) + } + + return out +} + +// Resolve reading conveniences +func (t *TrieNode) resolveTrieNodeExtension(p []string) (interface{}, []string, error) { + nibbles := t.elements[0].([]byte) + idx, rest := shiftFromPath(p, len(nibbles)) + if len(idx) < len(nibbles) { + return nil, nil, fmt.Errorf("not enough nibbles to traverse this extension") + } + + for _, i := range idx { + if getHexIndex(string(i)) == -1 { + return nil, nil, fmt.Errorf("invalid path element") + } + } + + for i, n := range nibbles { + if string(idx[i]) != fmt.Sprintf("%x", n) { + return nil, nil, fmt.Errorf("no such link in this extension") + } + } + + return &node.Link{Cid: t.elements[1].(cid.Cid)}, rest, nil +} + +func (t *TrieNode) resolveTrieNodeLeaf(p []string) (interface{}, []string, error) { + nibbles := t.elements[0].([]byte) + + if len(nibbles) != 0 { + idx, rest := shiftFromPath(p, len(nibbles)) + if len(idx) < len(nibbles) { + return nil, nil, fmt.Errorf("not enough nibbles to traverse this leaf") + } + + for _, i := range idx { + if getHexIndex(string(i)) == -1 { + return nil, nil, fmt.Errorf("invalid path element") + } + } + + for i, n := range nibbles { + if string(idx[i]) != fmt.Sprintf("%x", n) { + return nil, nil, fmt.Errorf("no such link in this extension") + } + } + + p = rest + } + + link, ok := t.elements[1].(node.Node) + if !ok { + return nil, nil, fmt.Errorf("leaf children is not an IPLD node") + } + + return link.Resolve(p) +} + +func (t *TrieNode) resolveTrieNodeBranch(p []string) (interface{}, []string, error) { + idx, rest := shiftFromPath(p, 1) + hidx := getHexIndex(idx) + if hidx == -1 { + return nil, nil, fmt.Errorf("incorrect path") + } + + child := t.elements[hidx] + if child != nil { + return &node.Link{Cid: child.(cid.Cid)}, rest, nil + } + return nil, nil, fmt.Errorf("no such link in this branch") +} + +// shiftFromPath extracts from a given path (as a slice of strings) +// the given number of elements as a single string, returning whatever +// it has not taken. +// +// Examples: +// ["0", "a", "something"] and 1 -> "0" and ["a", "something"] +// ["ab", "c", "d", "1"] and 2 -> "ab" and ["c", "d", "1"] +// ["abc", "d", "1"] and 2 -> "ab" and ["c", "d", "1"] +func shiftFromPath(p []string, i int) (string, []string) { + var ( + out string + rest []string + ) + + for _, pe := range p { + re := "" + for _, c := range pe { + if len(out) < i { + out += string(c) + } else { + re += string(c) + } + } + + if len(out) == i && re != "" { + rest = append(rest, re) + } + } + + return out, rest +} + +// getHexIndex returns to you the integer 0 - 15 equivalent to your +// string character if applicable, or -1 otherwise. +func getHexIndex(s string) int { + if len(s) != 1 { + return -1 + } + + c := byte(s[0]) + switch { + case '0' <= c && c <= '9': + return int(c - '0') + case 'a' <= c && c <= 'f': + return int(c - 'a' + 10) + } + + return -1 +} diff --git a/statediff/indexer/ipfs/models.go b/statediff/indexer/ipfs/models.go new file mode 100644 index 000000000..eb0312beb --- /dev/null +++ b/statediff/indexer/ipfs/models.go @@ -0,0 +1,22 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipfs + +type BlockModel struct { + CID string `db:"key"` + Data []byte `db:"data"` +} diff --git a/statediff/indexer/metrics.go b/statediff/indexer/metrics.go new file mode 100644 index 000000000..fc0727eda --- /dev/null +++ b/statediff/indexer/metrics.go @@ -0,0 +1,124 @@ +package indexer + +import ( + "database/sql" + "strings" + + "github.com/ethereum/go-ethereum/metrics" +) + +const ( + namespace = "statediff" +) + +// Build a fully qualified metric name +func metricName(subsystem, name string) string { + if name == "" { + return "" + } + parts := []string{namespace, name} + if subsystem != "" { + parts = []string{namespace, subsystem, name} + } + // Prometheus uses _ but geth metrics uses / and replaces + return strings.Join(parts, "/") +} + +type indexerMetricsHandles struct { + // The total number of processed blocks + blocks metrics.Counter + // The total number of processed transactions + transactions metrics.Counter + // The total number of processed receipts + receipts metrics.Counter + // Time spent waiting for free postgres tx + tFreePostgres metrics.Timer + // Postgres transaction commit duration + tPostgresCommit metrics.Timer + // Header processing time + tHeaderProcessing metrics.Timer + // Uncle processing time + tUncleProcessing metrics.Timer + // Tx and receipt processing time + tTxAndRecProcessing metrics.Timer + // State, storage, and code combined processing time + tStateStoreCodeProcessing metrics.Timer +} + +func RegisterIndexerMetrics(reg metrics.Registry) indexerMetricsHandles { + ctx := indexerMetricsHandles{ + blocks: metrics.NewCounter(), + transactions: metrics.NewCounter(), + receipts: metrics.NewCounter(), + tFreePostgres: metrics.NewTimer(), + tPostgresCommit: metrics.NewTimer(), + tHeaderProcessing: metrics.NewTimer(), + tUncleProcessing: metrics.NewTimer(), + tTxAndRecProcessing: metrics.NewTimer(), + tStateStoreCodeProcessing: metrics.NewTimer(), + } + subsys := "indexer" + reg.Register(metricName(subsys, "blocks"), ctx.blocks) + reg.Register(metricName(subsys, "transactions"), ctx.transactions) + reg.Register(metricName(subsys, "receipts"), ctx.receipts) + reg.Register(metricName(subsys, "t_free_postgres"), ctx.tFreePostgres) + reg.Register(metricName(subsys, "t_postgres_commit"), ctx.tPostgresCommit) + reg.Register(metricName(subsys, "t_header_processing"), ctx.tHeaderProcessing) + reg.Register(metricName(subsys, "t_uncle_processing"), ctx.tUncleProcessing) + reg.Register(metricName(subsys, "t_tx_receipt_processing"), ctx.tTxAndRecProcessing) + reg.Register(metricName(subsys, "t_state_store_code_processing"), ctx.tStateStoreCodeProcessing) + return ctx +} + +type dbMetricsHandles struct { + // Maximum number of open connections to the database + maxOpen metrics.Gauge + // The number of established connections both in use and idle + open metrics.Gauge + // The number of connections currently in use + inUse metrics.Gauge + // The number of idle connections + idle metrics.Gauge + // The total number of connections waited for + waitedFor metrics.Counter + // The total time blocked waiting for a new connection + blockedMilliseconds metrics.Counter + // The total number of connections closed due to SetMaxIdleConns + closedMaxIdle metrics.Counter + // The total number of connections closed due to SetConnMaxLifetime + closedMaxLifetime metrics.Counter +} + +func RegisterDBMetrics(reg metrics.Registry) dbMetricsHandles { + ctx := dbMetricsHandles{ + maxOpen: metrics.NewGauge(), + open: metrics.NewGauge(), + inUse: metrics.NewGauge(), + idle: metrics.NewGauge(), + waitedFor: metrics.NewCounter(), + blockedMilliseconds: metrics.NewCounter(), + closedMaxIdle: metrics.NewCounter(), + closedMaxLifetime: metrics.NewCounter(), + } + subsys := "connections" + reg.Register(metricName(subsys, "max_open"), ctx.maxOpen) + reg.Register(metricName(subsys, "open"), ctx.open) + reg.Register(metricName(subsys, "in_use"), ctx.inUse) + reg.Register(metricName(subsys, "idle"), ctx.idle) + reg.Register(metricName(subsys, "waited_for"), ctx.waitedFor) + reg.Register(metricName(subsys, "blocked_milliseconds"), ctx.blockedMilliseconds) + reg.Register(metricName(subsys, "closed_max_idle"), ctx.closedMaxIdle) + reg.Register(metricName(subsys, "closed_max_lifetime"), ctx.closedMaxLifetime) + return ctx +} + +func (met *dbMetricsHandles) Update(stats sql.DBStats) { + met.maxOpen.Update(int64(stats.MaxOpenConnections)) + met.open.Update(int64(stats.OpenConnections)) + met.inUse.Update(int64(stats.InUse)) + met.idle.Update(int64(stats.Idle)) + met.waitedFor.Inc(int64(stats.WaitCount)) + met.blockedMilliseconds.Inc(int64(stats.WaitDuration.Milliseconds())) + met.closedMaxIdle.Inc(int64(stats.MaxIdleClosed)) + met.closedMaxLifetime.Inc(int64(stats.MaxLifetimeClosed)) +} diff --git a/statediff/indexer/mocks/test_data.go b/statediff/indexer/mocks/test_data.go new file mode 100644 index 000000000..e2a2af54b --- /dev/null +++ b/statediff/indexer/mocks/test_data.go @@ -0,0 +1,194 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package mocks + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "math/big" + + "github.com/ethereum/go-ethereum/trie" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rlp" + "github.com/multiformats/go-multihash" + + "github.com/ethereum/go-ethereum/statediff/indexer/ipfs/ipld" + "github.com/ethereum/go-ethereum/statediff/testhelpers" + sdtypes "github.com/ethereum/go-ethereum/statediff/types" +) + +// Test variables +var ( + // block data + BlockNumber = big.NewInt(1) + MockHeader = types.Header{ + Time: 0, + Number: new(big.Int).Set(BlockNumber), + Root: common.HexToHash("0x0"), + TxHash: common.HexToHash("0x0"), + ReceiptHash: common.HexToHash("0x0"), + Difficulty: big.NewInt(5000000), + Extra: []byte{}, + } + MockTransactions, MockReceipts, SenderAddr = createTransactionsAndReceipts() + ReceiptsRlp, _ = rlp.EncodeToBytes(MockReceipts) + MockBlock = types.NewBlock(&MockHeader, MockTransactions, nil, MockReceipts, new(trie.Trie)) + MockHeaderRlp, _ = rlp.EncodeToBytes(MockBlock.Header()) + Address = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592") + AnotherAddress = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593") + ContractAddress = crypto.CreateAddress(SenderAddr, MockTransactions[2].Nonce()) + ContractHash = crypto.Keccak256Hash(ContractAddress.Bytes()).String() + MockContractByteCode = []byte{0, 1, 2, 3, 4, 5} + mockTopic11 = common.HexToHash("0x04") + mockTopic12 = common.HexToHash("0x06") + mockTopic21 = common.HexToHash("0x05") + mockTopic22 = common.HexToHash("0x07") + MockLog1 = &types.Log{ + Address: Address, + Topics: []common.Hash{mockTopic11, mockTopic12}, + Data: []byte{}, + } + MockLog2 = &types.Log{ + Address: AnotherAddress, + Topics: []common.Hash{mockTopic21, mockTopic22}, + Data: []byte{}, + } + HeaderCID, _ = ipld.RawdataToCid(ipld.MEthHeader, MockHeaderRlp, multihash.KECCAK_256) + Trx1CID, _ = ipld.RawdataToCid(ipld.MEthTx, MockTransactions.GetRlp(0), multihash.KECCAK_256) + Trx2CID, _ = ipld.RawdataToCid(ipld.MEthTx, MockTransactions.GetRlp(1), multihash.KECCAK_256) + Trx3CID, _ = ipld.RawdataToCid(ipld.MEthTx, MockTransactions.GetRlp(2), multihash.KECCAK_256) + Rct1CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, MockReceipts.GetRlp(0), multihash.KECCAK_256) + Rct2CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, MockReceipts.GetRlp(1), multihash.KECCAK_256) + Rct3CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, MockReceipts.GetRlp(2), multihash.KECCAK_256) + State1CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, ContractLeafNode, multihash.KECCAK_256) + State2CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, AccountLeafNode, multihash.KECCAK_256) + StorageCID, _ = ipld.RawdataToCid(ipld.MEthStorageTrie, StorageLeafNode, multihash.KECCAK_256) + + // statediff data + storageLocation = common.HexToHash("0") + StorageLeafKey = crypto.Keccak256Hash(storageLocation[:]).Bytes() + StorageValue = common.Hex2Bytes("01") + StoragePartialPath = common.Hex2Bytes("20290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563") + StorageLeafNode, _ = rlp.EncodeToBytes([]interface{}{ + StoragePartialPath, + StorageValue, + }) + + nonce1 = uint64(1) + ContractRoot = "0x821e2556a290c86405f8160a2d662042a431ba456b9db265c79bb837c04be5f0" + ContractCodeHash = common.HexToHash("0x753f98a8d4328b15636e46f66f2cb4bc860100aa17967cc145fcd17d1d4710ea") + ContractLeafKey = testhelpers.AddressToLeafKey(ContractAddress) + ContractAccount, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce1, + Balance: big.NewInt(0), + CodeHash: ContractCodeHash.Bytes(), + Root: common.HexToHash(ContractRoot), + }) + ContractPartialPath = common.Hex2Bytes("3114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45") + ContractLeafNode, _ = rlp.EncodeToBytes([]interface{}{ + ContractPartialPath, + ContractAccount, + }) + + nonce0 = uint64(0) + AccountRoot = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" + AccountCodeHash = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470") + AccountLeafKey = testhelpers.Account2LeafKey + Account, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce0, + Balance: big.NewInt(1000), + CodeHash: AccountCodeHash.Bytes(), + Root: common.HexToHash(AccountRoot), + }) + AccountPartialPath = common.Hex2Bytes("3957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45") + AccountLeafNode, _ = rlp.EncodeToBytes([]interface{}{ + AccountPartialPath, + Account, + }) + + StateDiffs = []sdtypes.StateNode{ + { + Path: []byte{'\x06'}, + NodeType: sdtypes.Leaf, + LeafKey: ContractLeafKey, + NodeValue: ContractLeafNode, + StorageNodes: []sdtypes.StorageNode{ + { + Path: []byte{}, + NodeType: sdtypes.Leaf, + LeafKey: StorageLeafKey, + NodeValue: StorageLeafNode, + }, + }, + }, + { + Path: []byte{'\x0c'}, + NodeType: sdtypes.Leaf, + LeafKey: AccountLeafKey, + NodeValue: AccountLeafNode, + StorageNodes: []sdtypes.StorageNode{}, + }, + } +) + +// createTransactionsAndReceipts is a helper function to generate signed mock transactions and mock receipts with mock logs +func createTransactionsAndReceipts() (types.Transactions, types.Receipts, common.Address) { + // make transactions + trx1 := types.NewTransaction(0, Address, big.NewInt(1000), 50, big.NewInt(100), []byte{}) + trx2 := types.NewTransaction(1, AnotherAddress, big.NewInt(2000), 100, big.NewInt(200), []byte{}) + trx3 := types.NewContractCreation(2, big.NewInt(1500), 75, big.NewInt(150), MockContractByteCode) + transactionSigner := types.MakeSigner(params.MainnetChainConfig, new(big.Int).Set(BlockNumber)) + mockCurve := elliptic.P256() + mockPrvKey, err := ecdsa.GenerateKey(mockCurve, rand.Reader) + if err != nil { + log.Crit(err.Error()) + } + signedTrx1, err := types.SignTx(trx1, transactionSigner, mockPrvKey) + if err != nil { + log.Crit(err.Error()) + } + signedTrx2, err := types.SignTx(trx2, transactionSigner, mockPrvKey) + if err != nil { + log.Crit(err.Error()) + } + signedTrx3, err := types.SignTx(trx3, transactionSigner, mockPrvKey) + if err != nil { + log.Crit(err.Error()) + } + SenderAddr, err := types.Sender(transactionSigner, signedTrx1) // same for both trx + if err != nil { + log.Crit(err.Error()) + } + // make receipts + mockReceipt1 := types.NewReceipt(common.HexToHash("0x0").Bytes(), false, 50) + mockReceipt1.Logs = []*types.Log{MockLog1} + mockReceipt1.TxHash = signedTrx1.Hash() + mockReceipt2 := types.NewReceipt(common.HexToHash("0x1").Bytes(), false, 100) + mockReceipt2.Logs = []*types.Log{MockLog2} + mockReceipt2.TxHash = signedTrx2.Hash() + mockReceipt3 := types.NewReceipt(common.HexToHash("0x2").Bytes(), false, 75) + mockReceipt3.Logs = []*types.Log{} + mockReceipt3.TxHash = signedTrx3.Hash() + return types.Transactions{signedTrx1, signedTrx2, signedTrx3}, types.Receipts{mockReceipt1, mockReceipt2, mockReceipt3}, SenderAddr +} diff --git a/statediff/indexer/models/models.go b/statediff/indexer/models/models.go new file mode 100644 index 000000000..8e8ebf624 --- /dev/null +++ b/statediff/indexer/models/models.go @@ -0,0 +1,126 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package models + +import "github.com/lib/pq" + +// HeaderModel is the db model for eth.header_cids +type HeaderModel struct { + ID int64 `db:"id"` + BlockNumber string `db:"block_number"` + BlockHash string `db:"block_hash"` + ParentHash string `db:"parent_hash"` + CID string `db:"cid"` + MhKey string `db:"mh_key"` + TotalDifficulty string `db:"td"` + NodeID int64 `db:"node_id"` + Reward string `db:"reward"` + StateRoot string `db:"state_root"` + UncleRoot string `db:"uncle_root"` + TxRoot string `db:"tx_root"` + RctRoot string `db:"receipt_root"` + Bloom []byte `db:"bloom"` + Timestamp uint64 `db:"timestamp"` + TimesValidated int64 `db:"times_validated"` +} + +// UncleModel is the db model for eth.uncle_cids +type UncleModel struct { + ID int64 `db:"id"` + HeaderID int64 `db:"header_id"` + BlockHash string `db:"block_hash"` + ParentHash string `db:"parent_hash"` + CID string `db:"cid"` + MhKey string `db:"mh_key"` + Reward string `db:"reward"` +} + +// TxModel is the db model for eth.transaction_cids +type TxModel struct { + ID int64 `db:"id"` + HeaderID int64 `db:"header_id"` + Index int64 `db:"index"` + TxHash string `db:"tx_hash"` + CID string `db:"cid"` + MhKey string `db:"mh_key"` + Dst string `db:"dst"` + Src string `db:"src"` + Data []byte `db:"tx_data"` + Deployment bool `db:"deployment"` +} + +// ReceiptModel is the db model for eth.receipt_cids +type ReceiptModel struct { + ID int64 `db:"id"` + TxID int64 `db:"tx_id"` + CID string `db:"cid"` + MhKey string `db:"mh_key"` + Contract string `db:"contract"` + ContractHash string `db:"contract_hash"` + LogContracts pq.StringArray `db:"log_contracts"` + Topic0s pq.StringArray `db:"topic0s"` + Topic1s pq.StringArray `db:"topic1s"` + Topic2s pq.StringArray `db:"topic2s"` + Topic3s pq.StringArray `db:"topic3s"` +} + +// StateNodeModel is the db model for eth.state_cids +type StateNodeModel struct { + ID int64 `db:"id"` + HeaderID int64 `db:"header_id"` + Path []byte `db:"state_path"` + StateKey string `db:"state_leaf_key"` + NodeType int `db:"node_type"` + CID string `db:"cid"` + MhKey string `db:"mh_key"` + Diff bool `db:"diff"` +} + +// StorageNodeModel is the db model for eth.storage_cids +type StorageNodeModel struct { + ID int64 `db:"id"` + StateID int64 `db:"state_id"` + Path []byte `db:"storage_path"` + StorageKey string `db:"storage_leaf_key"` + NodeType int `db:"node_type"` + CID string `db:"cid"` + MhKey string `db:"mh_key"` + Diff bool `db:"diff"` +} + +// StorageNodeWithStateKeyModel is a db model for eth.storage_cids + eth.state_cids.state_key +type StorageNodeWithStateKeyModel struct { + ID int64 `db:"id"` + StateID int64 `db:"state_id"` + Path []byte `db:"storage_path"` + StateKey string `db:"state_leaf_key"` + StorageKey string `db:"storage_leaf_key"` + NodeType int `db:"node_type"` + CID string `db:"cid"` + MhKey string `db:"mh_key"` + Diff bool `db:"diff"` +} + +// StateAccountModel is a db model for an eth state account (decoded value of state leaf node) +type StateAccountModel struct { + ID int64 `db:"id"` + StateID int64 `db:"state_id"` + Balance string `db:"balance"` + Nonce uint64 `db:"nonce"` + CodeHash []byte `db:"code_hash"` + StorageRoot string `db:"storage_root"` +} diff --git a/statediff/indexer/node/node.go b/statediff/indexer/node/node.go new file mode 100644 index 000000000..527546efa --- /dev/null +++ b/statediff/indexer/node/node.go @@ -0,0 +1,25 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package node + +type Info struct { + GenesisBlock string + NetworkID string + ChainID uint64 + ID string + ClientName string +} diff --git a/statediff/indexer/postgres/config.go b/statediff/indexer/postgres/config.go new file mode 100644 index 000000000..c2de0a6bf --- /dev/null +++ b/statediff/indexer/postgres/config.go @@ -0,0 +1,59 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package postgres + +import ( + "fmt" +) + +// Env variables +const ( + DATABASE_NAME = "DATABASE_NAME" + DATABASE_HOSTNAME = "DATABASE_HOSTNAME" + DATABASE_PORT = "DATABASE_PORT" + DATABASE_USER = "DATABASE_USER" + DATABASE_PASSWORD = "DATABASE_PASSWORD" + DATABASE_MAX_IDLE_CONNECTIONS = "DATABASE_MAX_IDLE_CONNECTIONS" + DATABASE_MAX_OPEN_CONNECTIONS = "DATABASE_MAX_OPEN_CONNECTIONS" + DATABASE_MAX_CONN_LIFETIME = "DATABASE_MAX_CONN_LIFETIME" +) + +type ConnectionParams struct { + Hostname string + Name string + User string + Password string + Port int +} + +type ConnectionConfig struct { + MaxIdle int + MaxOpen int + MaxLifetime int +} + +func DbConnectionString(params ConnectionParams) string { + if len(params.User) > 0 && len(params.Password) > 0 { + return fmt.Sprintf("postgresql://%s:%s@%s:%d/%s?sslmode=disable", + params.User, params.Password, params.Hostname, params.Port, params.Name) + } + if len(params.User) > 0 && len(params.Password) == 0 { + return fmt.Sprintf("postgresql://%s@%s:%d/%s?sslmode=disable", + params.User, params.Hostname, params.Port, params.Name) + } + return fmt.Sprintf("postgresql://%s:%d/%s?sslmode=disable", params.Hostname, params.Port, params.Name) +} diff --git a/statediff/indexer/postgres/errors.go b/statediff/indexer/postgres/errors.go new file mode 100644 index 000000000..f368f900b --- /dev/null +++ b/statediff/indexer/postgres/errors.go @@ -0,0 +1,53 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package postgres + +import ( + "fmt" +) + +const ( + BeginTransactionFailedMsg = "failed to begin transaction" + DbConnectionFailedMsg = "db connection failed" + DeleteQueryFailedMsg = "delete query failed" + InsertQueryFailedMsg = "insert query failed" + SettingNodeFailedMsg = "unable to set db node" +) + +func ErrBeginTransactionFailed(beginErr error) error { + return formatError(BeginTransactionFailedMsg, beginErr.Error()) +} + +func ErrDBConnectionFailed(connectErr error) error { + return formatError(DbConnectionFailedMsg, connectErr.Error()) +} + +func ErrDBDeleteFailed(deleteErr error) error { + return formatError(DeleteQueryFailedMsg, deleteErr.Error()) +} + +func ErrDBInsertFailed(insertErr error) error { + return formatError(InsertQueryFailedMsg, insertErr.Error()) +} + +func ErrUnableToSetNode(setErr error) error { + return formatError(SettingNodeFailedMsg, setErr.Error()) +} + +func formatError(msg, err string) error { + return fmt.Errorf("%s: %s", msg, err) +} diff --git a/statediff/indexer/postgres/postgres.go b/statediff/indexer/postgres/postgres.go new file mode 100644 index 000000000..455dac306 --- /dev/null +++ b/statediff/indexer/postgres/postgres.go @@ -0,0 +1,76 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package postgres + +import ( + "time" + + "github.com/jmoiron/sqlx" + _ "github.com/lib/pq" //postgres driver + + "github.com/ethereum/go-ethereum/statediff/indexer/node" +) + +type DB struct { + *sqlx.DB + Node node.Info + NodeID int64 +} + +func NewDB(connectString string, config ConnectionConfig, node node.Info) (*DB, error) { + db, connectErr := sqlx.Connect("postgres", connectString) + if connectErr != nil { + return &DB{}, ErrDBConnectionFailed(connectErr) + } + if config.MaxOpen > 0 { + db.SetMaxOpenConns(config.MaxOpen) + } + if config.MaxIdle > 0 { + db.SetMaxIdleConns(config.MaxIdle) + } + if config.MaxLifetime > 0 { + lifetime := time.Duration(config.MaxLifetime) * time.Second + db.SetConnMaxLifetime(lifetime) + } + pg := DB{DB: db, Node: node} + nodeErr := pg.CreateNode(&node) + if nodeErr != nil { + return &DB{}, ErrUnableToSetNode(nodeErr) + } + return &pg, nil +} + +func (db *DB) CreateNode(node *node.Info) error { + var nodeID int64 + err := db.QueryRow( + `INSERT INTO nodes (genesis_block, network_id, node_id, client_name, chain_id) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (genesis_block, network_id, node_id, chain_id) + DO UPDATE + SET genesis_block = $1, + network_id = $2, + node_id = $3, + client_name = $4, + chain_id = $5 + RETURNING id`, + node.GenesisBlock, node.NetworkID, node.ID, node.ClientName, node.ChainID).Scan(&nodeID) + if err != nil { + return ErrUnableToSetNode(err) + } + db.NodeID = nodeID + return nil +} diff --git a/statediff/indexer/postgres/postgres_suite_test.go b/statediff/indexer/postgres/postgres_suite_test.go new file mode 100644 index 000000000..e11e9ea77 --- /dev/null +++ b/statediff/indexer/postgres/postgres_suite_test.go @@ -0,0 +1,25 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package postgres_test + +import ( + "github.com/ethereum/go-ethereum/log" +) + +func init() { + log.Root().SetHandler(log.DiscardHandler()) +} diff --git a/statediff/indexer/postgres/postgres_test.go b/statediff/indexer/postgres/postgres_test.go new file mode 100644 index 000000000..f86055352 --- /dev/null +++ b/statediff/indexer/postgres/postgres_test.go @@ -0,0 +1,131 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package postgres_test + +import ( + "fmt" + "strings" + "testing" + + "math/big" + + "github.com/jmoiron/sqlx" + _ "github.com/lib/pq" + + "github.com/ethereum/go-ethereum/statediff/indexer/node" + "github.com/ethereum/go-ethereum/statediff/indexer/postgres" + "github.com/ethereum/go-ethereum/statediff/indexer/shared" +) + +var DBParams postgres.ConnectionParams + +func expectContainsSubstring(t *testing.T, full string, sub string) { + if !strings.Contains(full, sub) { + t.Fatalf("Expected \"%v\" to contain substring \"%v\"\n", full, sub) + } +} + +func TestPostgresDB(t *testing.T) { + var sqlxdb *sqlx.DB + + t.Run("connects to the database", func(t *testing.T) { + var err error + pgConfig := postgres.DbConnectionString(DBParams) + + sqlxdb, err = sqlx.Connect("postgres", pgConfig) + + if err != nil { + t.Fatal(err) + } + if sqlxdb == nil { + t.Fatal("DB is nil") + } + }) + + t.Run("serializes big.Int to db", func(t *testing.T) { + // postgres driver doesn't support go big.Int type + // various casts in golang uint64, int64, overflow for + // transaction value (in wei) even though + // postgres numeric can handle an arbitrary + // sized int, so use string representation of big.Int + // and cast on insert + + pgConnectString := postgres.DbConnectionString(DBParams) + db, err := sqlx.Connect("postgres", pgConnectString) + if err != nil { + t.Fatal(err) + } + if err != nil { + t.Fatal(err) + } + + bi := new(big.Int) + bi.SetString("34940183920000000000", 10) + shared.ExpectEqual(t, bi.String(), "34940183920000000000") + + defer db.Exec(`DROP TABLE IF EXISTS example`) + _, err = db.Exec("CREATE TABLE example ( id INTEGER, data NUMERIC )") + if err != nil { + t.Fatal(err) + } + + sqlStatement := ` + INSERT INTO example (id, data) + VALUES (1, cast($1 AS NUMERIC))` + _, err = db.Exec(sqlStatement, bi.String()) + if err != nil { + t.Fatal(err) + } + + var data string + err = db.QueryRow(`SELECT data FROM example WHERE id = 1`).Scan(&data) + if err != nil { + t.Fatal(err) + } + + shared.ExpectEqual(t, bi.String(), data) + actual := new(big.Int) + actual.SetString(data, 10) + shared.ExpectEqual(t, actual, bi) + }) + + t.Run("throws error when can't connect to the database", func(t *testing.T) { + invalidDatabase := postgres.ConnectionParams{} + node := node.Info{GenesisBlock: "GENESIS", NetworkID: "1", ID: "x123", ClientName: "geth"} + + _, err := postgres.NewDB(postgres.DbConnectionString(invalidDatabase), + postgres.ConnectionConfig{}, node) + + if err == nil { + t.Fatal("Expected an error") + } + + expectContainsSubstring(t, err.Error(), postgres.DbConnectionFailedMsg) + }) + + t.Run("throws error when can't create node", func(t *testing.T) { + badHash := fmt.Sprintf("x %s", strings.Repeat("1", 100)) + node := node.Info{GenesisBlock: badHash, NetworkID: "1", ID: "x123", ClientName: "geth"} + + _, err := postgres.NewDB(postgres.DbConnectionString(DBParams), postgres.ConnectionConfig{}, node) + + if err == nil { + t.Fatal("Expected an error") + } + expectContainsSubstring(t, err.Error(), postgres.SettingNodeFailedMsg) + }) +} diff --git a/statediff/indexer/reward.go b/statediff/indexer/reward.go new file mode 100644 index 000000000..47e3f17b9 --- /dev/null +++ b/statediff/indexer/reward.go @@ -0,0 +1,76 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package indexer + +import ( + "math/big" + + "github.com/ethereum/go-ethereum/core/types" +) + +func CalcEthBlockReward(header *types.Header, uncles []*types.Header, txs types.Transactions, receipts types.Receipts) *big.Int { + staticBlockReward := staticRewardByBlockNumber(header.Number.Uint64()) + transactionFees := calcEthTransactionFees(txs, receipts) + uncleInclusionRewards := calcEthUncleInclusionRewards(header, uncles) + tmp := transactionFees.Add(transactionFees, uncleInclusionRewards) + return tmp.Add(tmp, staticBlockReward) +} + +func CalcUncleMinerReward(blockNumber, uncleBlockNumber uint64) *big.Int { + staticBlockReward := staticRewardByBlockNumber(blockNumber) + rewardDiv8 := staticBlockReward.Div(staticBlockReward, big.NewInt(8)) + mainBlock := new(big.Int).SetUint64(blockNumber) + uncleBlock := new(big.Int).SetUint64(uncleBlockNumber) + uncleBlockPlus8 := uncleBlock.Add(uncleBlock, big.NewInt(8)) + uncleBlockPlus8MinusMainBlock := uncleBlockPlus8.Sub(uncleBlockPlus8, mainBlock) + return rewardDiv8.Mul(rewardDiv8, uncleBlockPlus8MinusMainBlock) +} + +func staticRewardByBlockNumber(blockNumber uint64) *big.Int { + staticBlockReward := new(big.Int) + //https://blog.ethereum.org/2017/10/12/byzantium-hf-announcement/ + if blockNumber >= 7280000 { + staticBlockReward.SetString("2000000000000000000", 10) + } else if blockNumber >= 4370000 { + staticBlockReward.SetString("3000000000000000000", 10) + } else { + staticBlockReward.SetString("5000000000000000000", 10) + } + return staticBlockReward +} + +func calcEthTransactionFees(txs types.Transactions, receipts types.Receipts) *big.Int { + transactionFees := new(big.Int) + for i, transaction := range txs { + receipt := receipts[i] + gasPrice := big.NewInt(transaction.GasPrice().Int64()) + gasUsed := big.NewInt(int64(receipt.GasUsed)) + transactionFee := gasPrice.Mul(gasPrice, gasUsed) + transactionFees = transactionFees.Add(transactionFees, transactionFee) + } + return transactionFees +} + +func calcEthUncleInclusionRewards(header *types.Header, uncles []*types.Header) *big.Int { + uncleInclusionRewards := new(big.Int) + for range uncles { + staticBlockReward := staticRewardByBlockNumber(header.Number.Uint64()) + staticBlockReward.Div(staticBlockReward, big.NewInt(32)) + uncleInclusionRewards.Add(uncleInclusionRewards, staticBlockReward) + } + return uncleInclusionRewards +} diff --git a/statediff/indexer/shared/chain_type.go b/statediff/indexer/shared/chain_type.go new file mode 100644 index 000000000..c3dedfe38 --- /dev/null +++ b/statediff/indexer/shared/chain_type.go @@ -0,0 +1,78 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package shared + +import ( + "errors" + "strings" +) + +// ChainType enum for specifying blockchain +type ChainType int + +const ( + UnknownChain ChainType = iota + Ethereum + Bitcoin + Omni + EthereumClassic +) + +func (c ChainType) String() string { + switch c { + case Ethereum: + return "Ethereum" + case Bitcoin: + return "Bitcoin" + case Omni: + return "Omni" + case EthereumClassic: + return "EthereumClassic" + default: + return "" + } +} + +func (c ChainType) API() string { + switch c { + case Ethereum: + return "eth" + case Bitcoin: + return "btc" + case Omni: + return "omni" + case EthereumClassic: + return "etc" + default: + return "" + } +} + +func NewChainType(name string) (ChainType, error) { + switch strings.ToLower(name) { + case "ethereum", "eth": + return Ethereum, nil + case "bitcoin", "btc", "xbt": + return Bitcoin, nil + case "omni": + return Omni, nil + case "classic", "etc": + return EthereumClassic, nil + default: + return UnknownChain, errors.New("invalid name for chain") + } +} diff --git a/statediff/indexer/shared/constants.go b/statediff/indexer/shared/constants.go new file mode 100644 index 000000000..3dc2994c4 --- /dev/null +++ b/statediff/indexer/shared/constants.go @@ -0,0 +1,22 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package shared + +const ( + DefaultMaxBatchSize uint64 = 100 + DefaultMaxBatchNumber int64 = 50 +) diff --git a/statediff/indexer/shared/data_type.go b/statediff/indexer/shared/data_type.go new file mode 100644 index 000000000..01fed57f7 --- /dev/null +++ b/statediff/indexer/shared/data_type.go @@ -0,0 +1,101 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package shared + +import ( + "fmt" + "strings" +) + +// DataType is an enum to loosely represent type of chain data +type DataType int + +const ( + UnknownDataType DataType = iota - 1 + Full + Headers + Uncles + Transactions + Receipts + State + Storage +) + +// String() method to resolve ReSyncType enum +func (r DataType) String() string { + switch r { + case Full: + return "full" + case Headers: + return "headers" + case Uncles: + return "uncles" + case Transactions: + return "transactions" + case Receipts: + return "receipts" + case State: + return "state" + case Storage: + return "storage" + default: + return "unknown" + } +} + +// GenerateDataTypeFromString +func GenerateDataTypeFromString(str string) (DataType, error) { + switch strings.ToLower(str) { + case "full", "f": + return Full, nil + case "headers", "header", "h": + return Headers, nil + case "uncles", "u": + return Uncles, nil + case "transactions", "transaction", "trxs", "txs", "trx", "tx", "t": + return Transactions, nil + case "receipts", "receipt", "rcts", "rct", "r": + return Receipts, nil + case "state": + return State, nil + case "storage": + return Storage, nil + default: + return UnknownDataType, fmt.Errorf("unrecognized resync type: %s", str) + } +} + +func SupportedDataType(d DataType) (bool, error) { + switch d { + case Full: + return true, nil + case Headers: + return true, nil + case Uncles: + return true, nil + case Transactions: + return true, nil + case Receipts: + return true, nil + case State: + return true, nil + case Storage: + return true, nil + default: + return true, nil + } +} diff --git a/statediff/indexer/shared/functions.go b/statediff/indexer/shared/functions.go new file mode 100644 index 000000000..257100289 --- /dev/null +++ b/statediff/indexer/shared/functions.go @@ -0,0 +1,124 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package shared + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/statediff/indexer/ipfs/ipld" + + "github.com/ipfs/go-cid" + "github.com/ipfs/go-ipfs-blockstore" + "github.com/ipfs/go-ipfs-ds-help" + node "github.com/ipfs/go-ipld-format" + "github.com/jmoiron/sqlx" + "github.com/multiformats/go-multihash" +) + +// HandleZeroAddrPointer will return an emtpy string for a nil address pointer +func HandleZeroAddrPointer(to *common.Address) string { + if to == nil { + return "" + } + return to.Hex() +} + +// HandleZeroAddr will return an empty string for a 0 value address +func HandleZeroAddr(to common.Address) string { + if to.Hex() == "0x0000000000000000000000000000000000000000" { + return "" + } + return to.Hex() +} + +// Rollback sql transaction and log any error +func Rollback(tx *sqlx.Tx) { + if err := tx.Rollback(); err != nil { + log.Error(err.Error()) + } +} + +// PublishIPLD is used to insert an IPLD into Postgres blockstore with the provided tx +func PublishIPLD(tx *sqlx.Tx, i node.Node) error { + dbKey := dshelp.MultihashToDsKey(i.Cid().Hash()) + prefixedKey := blockstore.BlockPrefix.String() + dbKey.String() + raw := i.RawData() + _, err := tx.Exec(`INSERT INTO public.blocks (key, data) VALUES ($1, $2) ON CONFLICT (key) DO NOTHING`, prefixedKey, raw) + return err +} + +// FetchIPLD is used to retrieve an ipld from Postgres blockstore with the provided tx and cid string +func FetchIPLD(tx *sqlx.Tx, cid string) ([]byte, error) { + mhKey, err := MultihashKeyFromCIDString(cid) + if err != nil { + return nil, err + } + pgStr := `SELECT data FROM public.blocks WHERE key = $1` + var block []byte + return block, tx.Get(&block, pgStr, mhKey) +} + +// FetchIPLDByMhKey is used to retrieve an ipld from Postgres blockstore with the provided tx and mhkey string +func FetchIPLDByMhKey(tx *sqlx.Tx, mhKey string) ([]byte, error) { + pgStr := `SELECT data FROM public.blocks WHERE key = $1` + var block []byte + return block, tx.Get(&block, pgStr, mhKey) +} + +// MultihashKeyFromCID converts a cid into a blockstore-prefixed multihash db key string +func MultihashKeyFromCID(c cid.Cid) string { + dbKey := dshelp.MultihashToDsKey(c.Hash()) + return blockstore.BlockPrefix.String() + dbKey.String() +} + +// MultihashKeyFromCIDString converts a cid string into a blockstore-prefixed multihash db key string +func MultihashKeyFromCIDString(c string) (string, error) { + dc, err := cid.Decode(c) + if err != nil { + return "", err + } + dbKey := dshelp.MultihashToDsKey(dc.Hash()) + return blockstore.BlockPrefix.String() + dbKey.String(), nil +} + +// PublishRaw derives a cid from raw bytes and provided codec and multihash type, and writes it to the db tx +func PublishRaw(tx *sqlx.Tx, codec, mh uint64, raw []byte) (string, error) { + c, err := ipld.RawdataToCid(codec, raw, mh) + if err != nil { + return "", err + } + dbKey := dshelp.MultihashToDsKey(c.Hash()) + prefixedKey := blockstore.BlockPrefix.String() + dbKey.String() + _, err = tx.Exec(`INSERT INTO public.blocks (key, data) VALUES ($1, $2) ON CONFLICT (key) DO NOTHING`, prefixedKey, raw) + return c.String(), err +} + +// MultihashKeyFromKeccak256 converts keccak256 hash bytes into a blockstore-prefixed multihash db key string +func MultihashKeyFromKeccak256(hash common.Hash) (string, error) { + mh, err := multihash.Encode(hash.Bytes(), multihash.KECCAK_256) + if err != nil { + return "", err + } + dbKey := dshelp.MultihashToDsKey(mh) + return blockstore.BlockPrefix.String() + dbKey.String(), nil +} + +// PublishDirect diretly writes a previously derived mhkey => value pair to the ipld database +func PublishDirect(tx *sqlx.Tx, key string, value []byte) error { + _, err := tx.Exec(`INSERT INTO public.blocks (key, data) VALUES ($1, $2) ON CONFLICT (key) DO NOTHING`, key, value) + return err +} diff --git a/statediff/indexer/shared/test_helpers.go b/statediff/indexer/shared/test_helpers.go new file mode 100644 index 000000000..b2e13f832 --- /dev/null +++ b/statediff/indexer/shared/test_helpers.go @@ -0,0 +1,68 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package shared + +import ( + "reflect" + "testing" + + "github.com/ipfs/go-cid" + "github.com/multiformats/go-multihash" + + "github.com/ethereum/go-ethereum/statediff/indexer/node" + "github.com/ethereum/go-ethereum/statediff/indexer/postgres" +) + +func ExpectEqual(t *testing.T, got interface{}, want interface{}) { + if !reflect.DeepEqual(got, want) { + t.Fatalf("Expected: %v\nActual: %v", want, got) + } +} + +// SetupDB is use to setup a db for watcher tests +func SetupDB() (*postgres.DB, error) { + uri := postgres.DbConnectionString(postgres.ConnectionParams{ + User: "postgres", + Password: "", + Hostname: "localhost", + Name: "vulcanize_testing", + Port: 5432, + }) + return postgres.NewDB(uri, postgres.ConnectionConfig{}, node.Info{}) +} + +// ListContainsString used to check if a list of strings contains a particular string +func ListContainsString(sss []string, s string) bool { + for _, str := range sss { + if s == str { + return true + } + } + return false +} + +// TestCID creates a basic CID for testing purposes +func TestCID(b []byte) cid.Cid { + pref := cid.Prefix{ + Version: 1, + Codec: cid.Raw, + MhType: multihash.KECCAK_256, + MhLength: -1, + } + c, _ := pref.Sum(b) + return c +} diff --git a/statediff/indexer/shared/types.go b/statediff/indexer/shared/types.go new file mode 100644 index 000000000..a9792522b --- /dev/null +++ b/statediff/indexer/shared/types.go @@ -0,0 +1,42 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package shared + +import "github.com/ethereum/go-ethereum/common" +import "github.com/ethereum/go-ethereum/statediff/types" +import "github.com/ethereum/go-ethereum/statediff/indexer/models" + +// Trie struct used to flag node as leaf or not +type TrieNode struct { + Path []byte + LeafKey common.Hash + Value []byte + Type types.NodeType +} + +// CIDPayload is a struct to hold all the CIDs and their associated meta data for indexing in Postgres +// Returned by IPLDPublisher +// Passed to CIDIndexer +type CIDPayload struct { + HeaderCID models.HeaderModel + UncleCIDs []models.UncleModel + TransactionCIDs []models.TxModel + ReceiptCIDs map[common.Hash]models.ReceiptModel + StateNodeCIDs []models.StateNodeModel + StateAccounts map[string]models.StateAccountModel + StorageNodeCIDs map[string][]models.StorageNodeModel +} diff --git a/statediff/indexer/test_helpers.go b/statediff/indexer/test_helpers.go new file mode 100644 index 000000000..024bb58f0 --- /dev/null +++ b/statediff/indexer/test_helpers.go @@ -0,0 +1,60 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package indexer + +import ( + "testing" + + "github.com/ethereum/go-ethereum/statediff/indexer/postgres" +) + +// TearDownDB is used to tear down the watcher dbs after tests +func TearDownDB(t *testing.T, db *postgres.DB) { + tx, err := db.Beginx() + if err != nil { + t.Fatal(err) + } + + _, err = tx.Exec(`DELETE FROM eth.header_cids`) + if err != nil { + t.Fatal(err) + } + _, err = tx.Exec(`DELETE FROM eth.transaction_cids`) + if err != nil { + t.Fatal(err) + } + _, err = tx.Exec(`DELETE FROM eth.receipt_cids`) + if err != nil { + t.Fatal(err) + } + _, err = tx.Exec(`DELETE FROM eth.state_cids`) + if err != nil { + t.Fatal(err) + } + _, err = tx.Exec(`DELETE FROM eth.storage_cids`) + if err != nil { + t.Fatal(err) + } + _, err = tx.Exec(`DELETE FROM blocks`) + if err != nil { + t.Fatal(err) + } + err = tx.Commit() + if err != nil { + t.Fatal(err) + } +} diff --git a/statediff/indexer/writer.go b/statediff/indexer/writer.go new file mode 100644 index 000000000..24bef7ed3 --- /dev/null +++ b/statediff/indexer/writer.go @@ -0,0 +1,137 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package indexer + +import ( + "github.com/jmoiron/sqlx" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/statediff/indexer/models" + "github.com/ethereum/go-ethereum/statediff/indexer/postgres" + "github.com/ethereum/go-ethereum/statediff/indexer/shared" +) + +var ( + nullHash = common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000") +) + +// Handles processing and writing of indexed IPLD objects to Postgres +type PostgresCIDWriter struct { + db *postgres.DB +} + +// NewPostgresCIDWriter creates a new pointer to a Indexer which satisfies the PostgresCIDWriter interface +func NewPostgresCIDWriter(db *postgres.DB) *PostgresCIDWriter { + return &PostgresCIDWriter{ + db: db, + } +} + +func (in *PostgresCIDWriter) upsertHeaderCID(tx *sqlx.Tx, header models.HeaderModel) (int64, error) { + var headerID int64 + err := tx.QueryRowx(`INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) + ON CONFLICT (block_number, block_hash) DO UPDATE SET (parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated) = ($3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, eth.header_cids.times_validated + 1) + RETURNING id`, + header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, header.TotalDifficulty, in.db.NodeID, header.Reward, header.StateRoot, header.TxRoot, + header.RctRoot, header.UncleRoot, header.Bloom, header.Timestamp, header.MhKey, 1).Scan(&headerID) + if err == nil { + indexerMetrics.blocks.Inc(1) + } + return headerID, err +} + +func (in *PostgresCIDWriter) upsertUncleCID(tx *sqlx.Tx, uncle models.UncleModel, headerID int64) error { + _, err := tx.Exec(`INSERT INTO eth.uncle_cids (block_hash, header_id, parent_hash, cid, reward, mh_key) VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT (header_id, block_hash) DO UPDATE SET (parent_hash, cid, reward, mh_key) = ($3, $4, $5, $6)`, + uncle.BlockHash, headerID, uncle.ParentHash, uncle.CID, uncle.Reward, uncle.MhKey) + return err +} + +func (in *PostgresCIDWriter) upsertTransactionAndReceiptCIDs(tx *sqlx.Tx, payload shared.CIDPayload, headerID int64) error { + for _, trxCidMeta := range payload.TransactionCIDs { + var txID int64 + err := tx.QueryRowx(`INSERT INTO eth.transaction_cids (header_id, tx_hash, cid, dst, src, index, mh_key, tx_data) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + ON CONFLICT (header_id, tx_hash) DO UPDATE SET (cid, dst, src, index, mh_key, tx_data) = ($3, $4, $5, $6, $7, $8) + RETURNING id`, + headerID, trxCidMeta.TxHash, trxCidMeta.CID, trxCidMeta.Dst, trxCidMeta.Src, trxCidMeta.Index, trxCidMeta.MhKey, trxCidMeta.Data).Scan(&txID) + if err != nil { + return err + } + indexerMetrics.transactions.Inc(1) + receiptCidMeta, ok := payload.ReceiptCIDs[common.HexToHash(trxCidMeta.TxHash)] + if ok { + if err := in.upsertReceiptCID(tx, receiptCidMeta, txID); err != nil { + return err + } + } + } + return nil +} + +func (in *PostgresCIDWriter) upsertTransactionCID(tx *sqlx.Tx, transaction models.TxModel, headerID int64) (int64, error) { + var txID int64 + err := tx.QueryRowx(`INSERT INTO eth.transaction_cids (header_id, tx_hash, cid, dst, src, index, mh_key, tx_data) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + ON CONFLICT (header_id, tx_hash) DO UPDATE SET (cid, dst, src, index, mh_key, tx_data) = ($3, $4, $5, $6, $7, $8) + RETURNING id`, + headerID, transaction.TxHash, transaction.CID, transaction.Dst, transaction.Src, transaction.Index, transaction.MhKey, transaction.Data).Scan(&txID) + if err == nil { + indexerMetrics.transactions.Inc(1) + } + return txID, err +} + +func (in *PostgresCIDWriter) upsertReceiptCID(tx *sqlx.Tx, rct models.ReceiptModel, txID int64) error { + _, err := tx.Exec(`INSERT INTO eth.receipt_cids (tx_id, cid, contract, contract_hash, topic0s, topic1s, topic2s, topic3s, log_contracts, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + ON CONFLICT (tx_id) DO UPDATE SET (cid, contract, contract_hash, topic0s, topic1s, topic2s, topic3s, log_contracts, mh_key) = ($2, $3, $4, $5, $6, $7, $8, $9, $10)`, + txID, rct.CID, rct.Contract, rct.ContractHash, rct.Topic0s, rct.Topic1s, rct.Topic2s, rct.Topic3s, rct.LogContracts, rct.MhKey) + if err == nil { + indexerMetrics.receipts.Inc(1) + } + return err +} + +func (in *PostgresCIDWriter) upsertStateCID(tx *sqlx.Tx, stateNode models.StateNodeModel, headerID int64) (int64, error) { + var stateID int64 + var stateKey string + if stateNode.StateKey != nullHash.String() { + stateKey = stateNode.StateKey + } + err := tx.QueryRowx(`INSERT INTO eth.state_cids (header_id, state_leaf_key, cid, state_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT (header_id, state_path) DO UPDATE SET (state_leaf_key, cid, node_type, diff, mh_key) = ($2, $3, $5, $6, $7) + RETURNING id`, + headerID, stateKey, stateNode.CID, stateNode.Path, stateNode.NodeType, true, stateNode.MhKey).Scan(&stateID) + return stateID, err +} + +func (in *PostgresCIDWriter) upsertStateAccount(tx *sqlx.Tx, stateAccount models.StateAccountModel, stateID int64) error { + _, err := tx.Exec(`INSERT INTO eth.state_accounts (state_id, balance, nonce, code_hash, storage_root) VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (state_id) DO UPDATE SET (balance, nonce, code_hash, storage_root) = ($2, $3, $4, $5)`, + stateID, stateAccount.Balance, stateAccount.Nonce, stateAccount.CodeHash, stateAccount.StorageRoot) + return err +} + +func (in *PostgresCIDWriter) upsertStorageCID(tx *sqlx.Tx, storageCID models.StorageNodeModel, stateID int64) error { + var storageKey string + if storageCID.StorageKey != nullHash.String() { + storageKey = storageCID.StorageKey + } + _, err := tx.Exec(`INSERT INTO eth.storage_cids (state_id, storage_leaf_key, cid, storage_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT (state_id, storage_path) DO UPDATE SET (storage_leaf_key, cid, node_type, diff, mh_key) = ($2, $3, $5, $6, $7)`, + stateID, storageKey, storageCID.CID, storageCID.Path, storageCID.NodeType, true, storageCID.MhKey) + return err +} diff --git a/statediff/mainnet_tests/block0_rlp b/statediff/mainnet_tests/block0_rlp new file mode 100644 index 0000000000000000000000000000000000000000..eb912911d159fcb7116d8bb9aee0f20f04a1bdcd GIT binary patch literal 540 zcmey#B>9s`WB~&Kut4^V?~8lKW2;uTZcX2I=8Dv1Ay?@sT_wUF7CQZPxH8#&3N`~4 zT>mlMf9slxn`<~{&$`Ok{r1o~uA*BPo2tY)Zy7}Jv$`w@dm%3_eI|6-|7O=`lMejI zi}rXRJu_$R0mdZ#t&fwpC=yh#lM$Q6BTB551?1g^CgF~Th6RGVU)=UHsrKucTqDz5 i@MP;-eqDFg%TGgWSKmh^`?h+u1H-La+RLaRU-Uhq0?W7E0f))urAHh4LTt%Cei$r$3@0SR&|VnN5tTvzjbOKu*B6$+WjzF9T#@_xfFQR|9*?g-8ocF5VbK=zF9i+jgo zt5&yeP2YCriqvHxSLrETCBh#TI{kIHGTD8~U46Mc&nBi9x&0wd>mySemVB_$S>Wk> z(=oY3P@rVWt$8Q*u$ay*mj27JUE7!M&R(BOZ#FwF2zwzfE`26++5cwOXOj;6$cy%P zAU!i@?E%Il{jHCawsml^6A=o#o5 zB&Q^so0*vF=Va!UR_Lea8|s-X&}W};bd}-iDUxyWDbCki*GHEg$zl2ZbE3GQWEani SzqdMe1piW#YFjn$zySc*K!fW5 literal 0 HcmV?d00001 diff --git a/statediff/mainnet_tests/block3_rlp b/statediff/mainnet_tests/block3_rlp new file mode 100644 index 0000000000000000000000000000000000000000..86f90a83a6e553ed112ceb9f361344f32f5a48b5 GIT binary patch literal 1079 zcmey#V)BzoV!@WF3$H&*GYnccsi1dBi?R962$2NQMGg9nla~mvxXn1ZAX|K=#jz*v zB{pTi{_P^Qn*`xzM@}fN+ zNYBh!dw?-Xf9vDqEs6vc>||sZ>7bSQUxWTX<|g5ehL+G>P6-R#Q%f@R%MA4l^bB;< zlFcnsjEz$Cb29TvEA-Rz4fRYGq(0Ppqx>&5^x2-WI0^@n(n$#;=+V1lV`Ku^zd#x z-M@H&>>1w|_m0O_t!~|#zU|BvsmnsH(o?!hgg-2F`s;9Ivip=1ug@I)nlXPx>fGi+ zW5=g!B~7MyERf4%W^Mc{QM{$XsiAew<;@4YR)i{i=rQp5`Y!qMK01? literal 0 HcmV?d00001 diff --git a/statediff/mainnet_tests/builder_test.go b/statediff/mainnet_tests/builder_test.go new file mode 100644 index 000000000..6412331ff --- /dev/null +++ b/statediff/mainnet_tests/builder_test.go @@ -0,0 +1,685 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package statediff_test + +import ( + "bytes" + "io/ioutil" + "log" + "math/big" + "os" + "sort" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/statediff" + "github.com/ethereum/go-ethereum/statediff/testhelpers" + sdtypes "github.com/ethereum/go-ethereum/statediff/types" +) + +var ( + db ethdb.Database + genesisBlock, block0, block1, block2, block3 *types.Block + block1CoinbaseAddr, block2CoinbaseAddr, block3CoinbaseAddr common.Address + block1CoinbaseHash, block2CoinbaseHash, block3CoinbaseHash common.Hash + builder statediff.Builder + emptyStorage = make([]sdtypes.StorageNode, 0) + + // block 1 data + block1CoinbaseAccount, _ = rlp.EncodeToBytes(state.Account{ + Nonce: 0, + Balance: big.NewInt(5000000000000000000), + CodeHash: testhelpers.NullCodeHash.Bytes(), + Root: testhelpers.EmptyContractRoot, + }) + block1CoinbaseLeafNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("38251692195afc818c92b485fcb8a4691af89cbe5a2ab557b83a4261be2a9a"), + block1CoinbaseAccount, + }) + block1CoinbaseLeafNodeHash = crypto.Keccak256(block1CoinbaseLeafNode) + block1x040bBranchNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("cc947d5ebb80600bad471f12c6ad5e4981e3525ecf8a2d982cc032536ae8b66d"), + common.Hex2Bytes("e80e52462e635a834e90e86ccf7673a6430384aac17004d626f4db831f0624bc"), + common.Hex2Bytes("59a8f11f60cb0a8488831f242da02944a26fd269d0608a44b8b873ded9e59e1b"), + common.Hex2Bytes("1ffb51e987e3cbd2e1dc1a64508d2e2b265477e21698b0d10fdf137f35027f40"), + []byte{}, + common.Hex2Bytes("ce5077f49a13ff8199d0e77715fdd7bfd6364774effcd5499bd93cba54b3c644"), + common.Hex2Bytes("f5146783c048e66ce1a776ae990b4255e5fba458ece77fcb83ff6e91d6637a88"), + common.Hex2Bytes("6a0558b6c38852e985cf01c2156517c1c6a1e64c787a953c347825f050b236c6"), + common.Hex2Bytes("56b6e93958b99aaae158cc2329e71a1865ba6f39c67b096922c5cf3ed86b0ae5"), + []byte{}, + common.Hex2Bytes("50d317a89a3405367d66668902f2c9f273a8d0d7d5d790dc516bca142f4a84af"), + common.Hex2Bytes("c72ca72750fdc1af3e6da5c7c5d82c54e4582f15b488a8aa1674058a99825dae"), + common.Hex2Bytes("e1a489df7b18cde818da6d38e235b026c2e61bcd3d34880b3ed0d67e0e4f0159"), + common.Hex2Bytes("b58d5062f2609fd2d68f00d14ab33fef2b373853877cf40bf64729e85b8fdc54"), + block1CoinbaseLeafNodeHash, + []byte{}, + []byte{}, + }) + block1x040bBranchNodeHash = crypto.Keccak256(block1x040bBranchNode) + block1x04BranchNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("a9317a59365ca09cefcd384018696590afffc432e35a97e8f85aa48907bf3247"), + common.Hex2Bytes("e0bc229254ce7a6a736c3953e570ab18b4a7f5f2a9aa3c3057b5f17d250a1cad"), + common.Hex2Bytes("a2484ec8884dbe0cf24ece99d67df0d1fe78992d67cc777636a817cb2ef205aa"), + common.Hex2Bytes("12b78d4078c607747f06bb88bd08f839eaae0e3ac6854e5f65867d4f78abb84e"), + common.Hex2Bytes("359a51862df5462e4cd302f69cb338512f21eb37ce0791b9a562e72ec48b7dbf"), + common.Hex2Bytes("13f8d617b6a734da9235b6ac80bdd7aeaff6120c39aa223638d88f22d4ba4007"), + common.Hex2Bytes("02055c6400e0ec3440a8bb8fdfd7d6b6c57b7bf83e37d7e4e983d416fdd8314e"), + common.Hex2Bytes("4b1cca9eb3e47e805e7f4c80671a9fcd589fd6ddbe1790c3f3e177e8ede01b9e"), + common.Hex2Bytes("70c3815efb23b986018089e009a38e6238b8850b3efd33831913ca6fa9240249"), + common.Hex2Bytes("7084699d2e72a193fd75bb6108ae797b4661696eba2d631d521fc94acc7b3247"), + common.Hex2Bytes("b2b3cd9f1e46eb583a6185d9a96b4e80125e3d75e6191fdcf684892ef52935cb"), + block1x040bBranchNodeHash, + common.Hex2Bytes("34d9ff0fee6c929424e52268dedbc596d10786e909c5a68d6466c2aba17387ce"), + common.Hex2Bytes("7484d5e44b6ee6b10000708c37e035b42b818475620f9316beffc46531d1eebf"), + common.Hex2Bytes("30c8a283adccf2742272563cd3d6710c89ba21eac0118bf5310cfb231bcca77f"), + common.Hex2Bytes("4bae8558d2385b8d3bc6e6ede20bdbc5dbb0b5384c316ba8985682f88d2e506d"), + []byte{}, + }) + block1x04BranchNodeHash = crypto.Keccak256(block1x04BranchNode) + block1RootBranchNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("90dcaf88c40c7bbc95a912cbdde67c175767b31173df9ee4b0d733bfdd511c43"), + common.Hex2Bytes("babe369f6b12092f49181ae04ca173fb68d1a5456f18d20fa32cba73954052bd"), + common.Hex2Bytes("473ecf8a7e36a829e75039a3b055e51b8332cbf03324ab4af2066bbd6fbf0021"), + common.Hex2Bytes("bbda34753d7aa6c38e603f360244e8f59611921d9e1f128372fec0d586d4f9e0"), + block1x04BranchNodeHash, + common.Hex2Bytes("a5f3f2f7542148c973977c8a1e154c4300fec92f755f7846f1b734d3ab1d90e7"), + common.Hex2Bytes("e823850f50bf72baae9d1733a36a444ab65d0a6faaba404f0583ce0ca4dad92d"), + common.Hex2Bytes("f7a00cbe7d4b30b11faea3ae61b7f1f2b315b61d9f6bd68bfe587ad0eeceb721"), + common.Hex2Bytes("7117ef9fc932f1a88e908eaead8565c19b5645dc9e5b1b6e841c5edbdfd71681"), + common.Hex2Bytes("69eb2de283f32c11f859d7bcf93da23990d3e662935ed4d6b39ce3673ec84472"), + common.Hex2Bytes("203d26456312bbc4da5cd293b75b840fc5045e493d6f904d180823ec22bfed8e"), + common.Hex2Bytes("9287b5c21f2254af4e64fca76acc5cd87399c7f1ede818db4326c98ce2dc2208"), + common.Hex2Bytes("6fc2d754e304c48ce6a517753c62b1a9c1d5925b89707486d7fc08919e0a94ec"), + common.Hex2Bytes("7b1c54f15e299bd58bdfef9741538c7828b5d7d11a489f9c20d052b3471df475"), + common.Hex2Bytes("51f9dd3739a927c89e357580a4c97b40234aa01ed3d5e0390dc982a7975880a0"), + common.Hex2Bytes("89d613f26159af43616fd9455bb461f4869bfede26f2130835ed067a8b967bfb"), + []byte{}, + }) + + // block 2 data + block2CoinbaseAccount, _ = rlp.EncodeToBytes(state.Account{ + Nonce: 0, + Balance: big.NewInt(5000000000000000000), + CodeHash: testhelpers.NullCodeHash.Bytes(), + Root: testhelpers.EmptyContractRoot, + }) + block2CoinbaseLeafNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("20679cbcf198c1741a6f4e4473845659a30caa8b26f8d37a0be2e2bc0d8892"), + block2CoinbaseAccount, + }) + block2CoinbaseLeafNodeHash = crypto.Keccak256(block2CoinbaseLeafNode) + block2MovedPremineBalance, _ = new(big.Int).SetString("4000000000000000000000", 10) + block2MovedPremineAccount, _ = rlp.EncodeToBytes(state.Account{ + Nonce: 0, + Balance: block2MovedPremineBalance, + CodeHash: testhelpers.NullCodeHash.Bytes(), + Root: testhelpers.EmptyContractRoot, + }) + block2MovedPremineLeafNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("20f2e24db7943eab4415f99e109698863b0fecca1cf9ffc500f38cefbbe29e"), + block2MovedPremineAccount, + }) + block2MovedPremineLeafNodeHash = crypto.Keccak256(block2MovedPremineLeafNode) + block2x00080dBranchNode, _ = rlp.EncodeToBytes([]interface{}{ + block2MovedPremineLeafNodeHash, + []byte{}, + []byte{}, + []byte{}, + block2CoinbaseLeafNodeHash, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + }) + block2x00080dBranchNodeHash = crypto.Keccak256(block2x00080dBranchNode) + block2x0008BranchNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("def97a26f824fc3911cf7f8c41dfc9bc93cc36ae2248de22ecae01d6950b2dc9"), + common.Hex2Bytes("234a575e2c5badab8de0f6515b6723195323a0562fbe1316255888637043f1c1"), + common.Hex2Bytes("29659740af1c23306ee8f8294c71a5632ace8c80b1eb61cfdf7022f47ff52305"), + common.Hex2Bytes("cf2681d23bb666d89dec8123bce9e626240a7e2ce7a1e8316b1ee88181c9471c"), + common.Hex2Bytes("18d8de6967fe34b9fd411c74fecc45f8a737961791e70d8ece967bb07cf4d4dc"), + common.Hex2Bytes("7cad60c7cbca8c79c2db5a8fc1baa9381484d43d6c37dfb97718c3a109d47dfc"), + common.Hex2Bytes("2138f5a9062b750b6320e5fac5b134da90a9edbda06ef3e1ae64fb1366ca998c"), + common.Hex2Bytes("532826502a9661fcae7c0f5d2a4c8cb287dfc521e828349543c5a461a9d591ed"), + common.Hex2Bytes("30543537413dd086d4b1560f46b90e8da0f43de5584a138ab036d74e84657523"), + common.Hex2Bytes("c98042928af640bfa1142aca895cd76e146332dce94ddad3426e74ed519ca1e0"), + common.Hex2Bytes("43de3e62cc3148193899d018dff813c04c5b636ce95bd7e828416204292d9ff9"), + []byte{}, + common.Hex2Bytes("78d533b9182bb42f6c16e9ebd5734f0d280179ba1c9b6316c2c1df73f7dd8a54"), + block2x00080dBranchNodeHash, + common.Hex2Bytes("934b736b57a892aaa15a03c7e37746bb096313727135f9841cb64c263785cf81"), + common.Hex2Bytes("38ce97150e90dfd7258901a0ddee72d8e30760a3d0419dbb80135c66588739a2"), + []byte{}, + }) + block2x0008BranchNodeHash = crypto.Keccak256(block2x0008BranchNode) + block2x00BranchNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("e45a9e85cab1b6eb18b30df2c6acc448bbac6a30d81646823b31223e16e5063e"), + common.Hex2Bytes("33bd7171d556b981f6849064eb09412b24fedc0812127db936067043f53db1b9"), + common.Hex2Bytes("ca56945f074da4f15587404593faf3a50d17ea0e21a418ad6ec99bdf4bf3f914"), + common.Hex2Bytes("da23e9004f782df128eea1adff77952dc85f91b7f7ca4893aac5f21d24c3a1c9"), + common.Hex2Bytes("ba5ec61fa780ee02af19db99677c37560fc4f0df5c278d9dfa2837f30f72bc6b"), + common.Hex2Bytes("8310ad91625c2e3429a74066b7e2e0c958325e4e7fa3ec486b73b7c8300cfef7"), + common.Hex2Bytes("732e5c103bf4d5adfef83773026809d9405539b67e93293a02342e83ad2fb766"), + common.Hex2Bytes("30d14ff0c2aab57d1fbaf498ab14519b4e9d94f149a3dc15f0eec5adf8df25e1"), + block2x0008BranchNodeHash, + common.Hex2Bytes("5a43bd92e55aa78df60e70b6b53b6366c4080fd6a5bdd7b533b46aff4a75f6f2"), + common.Hex2Bytes("a0c410aa59efe416b1213166fab680ce330bd46c3ebf877ff14609ee6a383600"), + common.Hex2Bytes("2f41e918786e557293068b1eda9b3f9f86ed4e65a6a5363ee3262109f6e08b17"), + common.Hex2Bytes("01f42a40f02f6f24bb97b09c4d3934e8b03be7cfbb902acc1c8fd67a7a5abace"), + common.Hex2Bytes("0acbdce2787a6ea177209bd13bfc9d0779d7e2b5249e0211a2974164e14312f5"), + common.Hex2Bytes("dadbe113e4132e0c0c3cd4867e0a2044d0e5a3d44b350677ed42fc9244d004d4"), + common.Hex2Bytes("aa7441fefc17d76aedfcaf692fe71014b94c1547b6d129562b34fc5995ca0d1a"), + []byte{}, + }) + block2x00BranchNodeHash = crypto.Keccak256(block2x00BranchNode) + block2RootBranchNode, _ = rlp.EncodeToBytes([]interface{}{ + block2x00BranchNodeHash, + common.Hex2Bytes("babe369f6b12092f49181ae04ca173fb68d1a5456f18d20fa32cba73954052bd"), + common.Hex2Bytes("473ecf8a7e36a829e75039a3b055e51b8332cbf03324ab4af2066bbd6fbf0021"), + common.Hex2Bytes("bbda34753d7aa6c38e603f360244e8f59611921d9e1f128372fec0d586d4f9e0"), + block1x04BranchNodeHash, + common.Hex2Bytes("a5f3f2f7542148c973977c8a1e154c4300fec92f755f7846f1b734d3ab1d90e7"), + common.Hex2Bytes("e823850f50bf72baae9d1733a36a444ab65d0a6faaba404f0583ce0ca4dad92d"), + common.Hex2Bytes("f7a00cbe7d4b30b11faea3ae61b7f1f2b315b61d9f6bd68bfe587ad0eeceb721"), + common.Hex2Bytes("7117ef9fc932f1a88e908eaead8565c19b5645dc9e5b1b6e841c5edbdfd71681"), + common.Hex2Bytes("69eb2de283f32c11f859d7bcf93da23990d3e662935ed4d6b39ce3673ec84472"), + common.Hex2Bytes("203d26456312bbc4da5cd293b75b840fc5045e493d6f904d180823ec22bfed8e"), + common.Hex2Bytes("9287b5c21f2254af4e64fca76acc5cd87399c7f1ede818db4326c98ce2dc2208"), + common.Hex2Bytes("6fc2d754e304c48ce6a517753c62b1a9c1d5925b89707486d7fc08919e0a94ec"), + common.Hex2Bytes("7b1c54f15e299bd58bdfef9741538c7828b5d7d11a489f9c20d052b3471df475"), + common.Hex2Bytes("51f9dd3739a927c89e357580a4c97b40234aa01ed3d5e0390dc982a7975880a0"), + common.Hex2Bytes("89d613f26159af43616fd9455bb461f4869bfede26f2130835ed067a8b967bfb"), + []byte{}, + }) + + // block3 data + // path 060e0f + blcok3CoinbaseBalance, _ = new(big.Int).SetString("5156250000000000000", 10) + block3CoinbaseAccount, _ = rlp.EncodeToBytes(state.Account{ + Nonce: 0, + Balance: blcok3CoinbaseBalance, + CodeHash: testhelpers.NullCodeHash.Bytes(), + Root: testhelpers.EmptyContractRoot, + }) + block3CoinbaseLeafNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("3a174f00e64521a535f35e67c1aa241951c791639b2f3d060f49c5d9fa8b9e"), + block3CoinbaseAccount, + }) + block3CoinbaseLeafNodeHash = crypto.Keccak256(block3CoinbaseLeafNode) + // path 0c0e050703 + block3MovedPremineBalance1, _ = new(big.Int).SetString("3750000000000000000", 10) + block3MovedPremineAccount1, _ = rlp.EncodeToBytes(state.Account{ + Nonce: 0, + Balance: block3MovedPremineBalance1, + CodeHash: testhelpers.NullCodeHash.Bytes(), + Root: testhelpers.EmptyContractRoot, + }) + block3MovedPremineLeafNode1, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("3ced93917e658d10e2d9009470dad72b63c898d173721194a12f2ae5e190"), // ce573ced93917e658d10e2d9009470dad72b63c898d173721194a12f2ae5e190 + block3MovedPremineAccount1, + }) + block3MovedPremineLeafNodeHash1 = crypto.Keccak256(block3MovedPremineLeafNode1) + // path 0c0e050708 + block3MovedPremineBalance2, _ = new(big.Int).SetString("1999944000000000000000", 10) + block3MovedPremineAccount2, _ = rlp.EncodeToBytes(state.Account{ + Nonce: 0, + Balance: block3MovedPremineBalance2, + CodeHash: testhelpers.NullCodeHash.Bytes(), + Root: testhelpers.EmptyContractRoot, + }) + block3MovedPremineLeafNode2, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("33bc1e69eedf90f402e11f6862da14ed8e50156635a04d6393bbae154012"), // ce5783bc1e69eedf90f402e11f6862da14ed8e50156635a04d6393bbae154012 + block3MovedPremineAccount2, + }) + block3MovedPremineLeafNodeHash2 = crypto.Keccak256(block3MovedPremineLeafNode2) + + block3x0c0e0507BranchNode, _ = rlp.EncodeToBytes([]interface{}{ + []byte{}, + []byte{}, + []byte{}, + block3MovedPremineLeafNodeHash1, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + block3MovedPremineLeafNodeHash2, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + }) + block3x0c0e0507BranchNodeHash = crypto.Keccak256(block3x0c0e0507BranchNode) + + block3x0c0e05BranchNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("452e3beb503b1d87ae7c672b98a8e3fd043a671405502562ae1043dc97151a50"), + []byte{}, + common.Hex2Bytes("2f5bb16f77086f67ce8c4258cb9061cb299e597b2ad4ad6d7ccc474d6d88e85e"), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + block3x0c0e0507BranchNodeHash, + []byte{}, + common.Hex2Bytes("44623e5a9319f83870db0ea4611a25fca1e1da3eeea2be4a091dfc15ab45689e"), + common.Hex2Bytes("b41e047a97f44fa4cb8146467b88c8f4705811029d9e170abb0aba7d0af9f0da"), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + }) + block3x0c0e05BranchNodeHash = crypto.Keccak256(block3x0c0e05BranchNode) + + block3x060eBranchNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("94d77c7c30b88829c9989948b206cda5e532b38b49534261c517aebf4a3e6fdb"), + common.Hex2Bytes("a5cf57a50da8204964e834a12a53f9bed7afc9b700a4a81b440122d60c7603a7"), + []byte{}, + common.Hex2Bytes("3730ec0571f34b6c3b178dc26ccb31a3f50c29da9b1921e41b9477ddab41b0fe"), + []byte{}, + common.Hex2Bytes("543952bb9566c2018cf8d7b90d6a7903cdfff3d79ac36189be5322de42fc3fc0"), + []byte{}, + common.Hex2Bytes("c4a49b66f0bcc08531e50cdea5577a281d111fa542eaefd9a9aead8febb0735e"), + common.Hex2Bytes("362ad58916c71463b98c079649fc486c5f082c4f548bd4ab501515f0c5641cb4"), + common.Hex2Bytes("36aae109f6f55f0bd05eb05bb365af2332dfe5f06d3d17903e88534c319eb709"), + common.Hex2Bytes("430dcfc5cc49a6b490dd54138920e8f94e427239c2bccc14705cfd4ff6cc4383"), + common.Hex2Bytes("73ed77563dfed2fdb38900b474db88b2270f449167e0d877fda9e2229f119fe8"), + common.Hex2Bytes("5dfe06013f2a41f1779194ceb07769d019f518b2a694a82fa1661e60fd973eaa"), + common.Hex2Bytes("80bdfd85fbb6b45850bad6e34136aaa1b04711e47469fa2f0d19eca52089efb5"), + []byte{}, + block3CoinbaseLeafNodeHash, + []byte{}, + }) + block3x060eBranchNodeHash = crypto.Keccak256(block3x060eBranchNode) + + block3x0c0eBranchNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("70647f11b2b995d718f9e8aceb44c8839e0055641930d216fa6090280a9d63d5"), + common.Hex2Bytes("fdfb17cd2fba2a14219981cb7886a1977cd85dbef5c767c562f4a5f547febff0"), + common.Hex2Bytes("ff87313253ec6f860142b7bf62efb4cb07ea668c57aa90cbe9ef22b72fee15c7"), + common.Hex2Bytes("3a77b3c26a54ad37bdf4e19c1bce93493ec0f79d9ad90190b70bc840b54918e1"), + common.Hex2Bytes("af1b3b14324561b68f2e24dbcc28673ab35ce3fd0230fe2bc86b3d1931745195"), + block3x0c0e05BranchNodeHash, + common.Hex2Bytes("647dcbfe6aabcd9d219ff40422af4326bfc1ec66703195a78eb48618ddef248d"), + common.Hex2Bytes("2d2bf06159cc8928283c3419a03f08ea34c493a9d002a0ec76d5c429508ccaf4"), + common.Hex2Bytes("d7147251b3f48f25e1e4c6d8f83a00b1eca66e99a4ea0d238942ce72d0ba6414"), + common.Hex2Bytes("cb859370869967594fb29f4e2904413310146733d7fcbd11407f3e47626e0e34"), + common.Hex2Bytes("b93ab9b0bd83963860fbe0b7d543879cfde756ea1618d2a40d85483058cc5a26"), + common.Hex2Bytes("45aee096499d209931457ce251c5c7e5543f22524f67785ff8f0f3f02588b0ed"), + []byte{}, + common.Hex2Bytes("aa2ae9379797c5066bba646108074ae8677e82c923d584b6d1c1268ca3708c5c"), + common.Hex2Bytes("e6eb055f0d8e194c083471479a3de87fa0f90c0f4aaa518416ec1e469ec32e3a"), + common.Hex2Bytes("0cc9c50fc7eba162fb17f2e04e3599c13abbf210d9781864d0edec401ecaebba"), + []byte{}, + }) + block3x0c0eBranchNodeHash = crypto.Keccak256(block3x0c0eBranchNode) + + block3x06BranchNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("68f7ff8c074d6e4cccd55b5b1c2116a6dd7047d4332090e6db8839362991b0ae"), + common.Hex2Bytes("c446eb4377c750701374c56e50759e6ba68b7adf4d543e718c8b28a99ae3b6ad"), + common.Hex2Bytes("ef2c49ec64cb65eae0d99684e74c8af2bd0206c9a0214d9d3eddf0881dd8412a"), + common.Hex2Bytes("7096c4cc7e8125f0b142d8644ad681f8a8142e210c806f33f3f7004f0e9d6002"), + common.Hex2Bytes("bc9a8ae647b234cd6607b6b0245e3b3d5ec4f7ea006e7eda1f92d02f0ea91116"), + common.Hex2Bytes("a87720deb92ff2f899e809befab9970a61c86148c4fa09d04b77505ee4a5bda5"), + common.Hex2Bytes("2460e5b6ded7c0001de29c15db124614432fef6486370cc9970f63b0d95fd5e2"), + common.Hex2Bytes("ed1c447d4a32bc31e9e32259dc63da10df91231e786332e3df122b301b1f8fc3"), + common.Hex2Bytes("0d27dfc201d995c2323b792860dbca087da7cc56d1698c39b7c4b9277729c5ca"), + common.Hex2Bytes("f6d2be168d9c17643c9ea80c29322b364604cdfd36eef40123d83fad364e43fa"), + common.Hex2Bytes("004bf1c30a5730f464de1a0ba4ac5b5618df66d6106073d08742166e33a7eeb5"), + common.Hex2Bytes("7298d019a57a1b04ac31ed874d654ba0d3c249704c5d9efa1d08959fc89e0779"), + common.Hex2Bytes("fb3d50b7af6f839e371ff8ebd0322e94e6b6fb7888416737f88cf55bcf5859ec"), + common.Hex2Bytes("4e7a2618fa1fc560a73c24839657adf7e48d600ecfb12333678115936597a913"), + block3x060eBranchNodeHash, + common.Hex2Bytes("1909706c5db040f54c19f4050659ad484982145b02474653917de379f15ebb36"), + []byte{}, + }) + block3x06BranchNodeHash = crypto.Keccak256(block3x06BranchNode) + + block3x0cBranchNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("dae48f5b47930c28bb116fbd55e52cd47242c71bf55373b55eb2805ee2e4a929"), + common.Hex2Bytes("0f1f37f337ec800e2e5974e2e7355f10f1a4832b39b846d916c3597a460e0676"), + common.Hex2Bytes("da8f627bb8fbeead17b318e0a8e4f528db310f591bb6ab2deda4a9f7ca902ab5"), + common.Hex2Bytes("971c662648d58295d0d0aa4b8055588da0037619951217c22052802549d94a2f"), + common.Hex2Bytes("ccc701efe4b3413fd6a61a6c9f40e955af774649a8d9fd212d046a5a39ddbb67"), + common.Hex2Bytes("d607cdb32e2bd635ee7f2f9e07bc94ddbd09b10ec0901b66628e15667aec570b"), + common.Hex2Bytes("5b89203dc940e6fa70ec19ad4e01d01849d3a5baa0a8f9c0525256ed490b159f"), + common.Hex2Bytes("b84227d48df68aecc772939a59afa9e1a4ab578f7b698bdb1289e29b6044668e"), + common.Hex2Bytes("fd1c992070b94ace57e48cbf6511a16aa770c645f9f5efba87bbe59d0a042913"), + common.Hex2Bytes("e16a7ccea6748ae90de92f8aef3b3dc248a557b9ac4e296934313f24f7fced5f"), + common.Hex2Bytes("42373cf4a00630d94de90d0a23b8f38ced6b0f7cb818b8925fee8f0c2a28a25a"), + common.Hex2Bytes("5f89d2161c1741ff428864f7889866484cef622de5023a46e795dfdec336319f"), + common.Hex2Bytes("7597a017664526c8c795ce1da27b8b72455c49657113e0455552dbc068c5ba31"), + common.Hex2Bytes("d5be9089012fda2c585a1b961e988ea5efcd3a06988e150a8682091f694b37c5"), + block3x0c0eBranchNodeHash, + common.Hex2Bytes("49bf6e8df0acafd0eff86defeeb305568e44d52d2235cf340ae15c6034e2b241"), + []byte{}, + }) + block3x0cBranchNodeHash = crypto.Keccak256(block3x0cBranchNode) + + block3RootBranchNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("f646da473c426e79f1c796b00d4873f47de1dbe1c9d19d63993a05eeb8b4041d"), + common.Hex2Bytes("babe369f6b12092f49181ae04ca173fb68d1a5456f18d20fa32cba73954052bd"), + common.Hex2Bytes("473ecf8a7e36a829e75039a3b055e51b8332cbf03324ab4af2066bbd6fbf0021"), + common.Hex2Bytes("bbda34753d7aa6c38e603f360244e8f59611921d9e1f128372fec0d586d4f9e0"), + common.Hex2Bytes("d9cff5d5f2418afd16a4da5c221fdc8bd47520c5927922f69a68177b64da6ac0"), + common.Hex2Bytes("a5f3f2f7542148c973977c8a1e154c4300fec92f755f7846f1b734d3ab1d90e7"), + block3x06BranchNodeHash, + common.Hex2Bytes("f7a00cbe7d4b30b11faea3ae61b7f1f2b315b61d9f6bd68bfe587ad0eeceb721"), + common.Hex2Bytes("7117ef9fc932f1a88e908eaead8565c19b5645dc9e5b1b6e841c5edbdfd71681"), + common.Hex2Bytes("69eb2de283f32c11f859d7bcf93da23990d3e662935ed4d6b39ce3673ec84472"), + common.Hex2Bytes("203d26456312bbc4da5cd293b75b840fc5045e493d6f904d180823ec22bfed8e"), + common.Hex2Bytes("9287b5c21f2254af4e64fca76acc5cd87399c7f1ede818db4326c98ce2dc2208"), + block3x0cBranchNodeHash, + common.Hex2Bytes("7b1c54f15e299bd58bdfef9741538c7828b5d7d11a489f9c20d052b3471df475"), + common.Hex2Bytes("51f9dd3739a927c89e357580a4c97b40234aa01ed3d5e0390dc982a7975880a0"), + common.Hex2Bytes("89d613f26159af43616fd9455bb461f4869bfede26f2130835ed067a8b967bfb"), + []byte{}, + }) +) + +func init() { + db = rawdb.NewMemoryDatabase() + genesisBlock = core.DefaultGenesisBlock().MustCommit(db) + genBy, err := rlp.EncodeToBytes(genesisBlock) + if err != nil { + log.Fatal(err) + } + var block0RLP []byte + block0, block0RLP, err = loadBlockFromRLPFile("./block0_rlp") + if err != nil { + log.Fatal(err) + } + if !bytes.Equal(genBy, block0RLP) { + log.Fatal("mainnet genesis blocks do not match") + } + block1, _, err = loadBlockFromRLPFile("./block1_rlp") + if err != nil { + log.Fatal(err) + } + block1CoinbaseAddr = block1.Coinbase() + block1CoinbaseHash = crypto.Keccak256Hash(block1CoinbaseAddr.Bytes()) + block2, _, err = loadBlockFromRLPFile("./block2_rlp") + if err != nil { + log.Fatal(err) + } + block2CoinbaseAddr = block2.Coinbase() + block2CoinbaseHash = crypto.Keccak256Hash(block2CoinbaseAddr.Bytes()) + block3, _, err = loadBlockFromRLPFile("./block3_rlp") + if err != nil { + log.Fatal(err) + } + block3CoinbaseAddr = block3.Coinbase() + block3CoinbaseHash = crypto.Keccak256Hash(block3CoinbaseAddr.Bytes()) +} + +func loadBlockFromRLPFile(filename string) (*types.Block, []byte, error) { + f, err := os.Open(filename) + if err != nil { + return nil, nil, err + } + defer f.Close() + blockRLP, err := ioutil.ReadAll(f) + if err != nil { + return nil, nil, err + } + block := new(types.Block) + return block, blockRLP, rlp.DecodeBytes(blockRLP, block) +} + +func TestBuilderOnMainnetBlocks(t *testing.T) { + chain, _ := core.NewBlockChain(db, nil, params.MainnetChainConfig, ethash.NewFaker(), vm.Config{}, nil) + _, err := chain.InsertChain([]*types.Block{block1, block2, block3}) + if err != nil { + t.Error(err) + } + params := statediff.Params{ + IntermediateStateNodes: true, + } + builder = statediff.NewBuilder(chain.StateCache()) + + var tests = []struct { + name string + startingArguments statediff.Args + expected *statediff.StateObject + }{ + // note that block0 (genesis) has over 1000 nodes due to the pre-allocation for the crowd-sale + // it is not feasible to write a unit test of that size at this time + { + "testBlock1", + //10000 transferred from testBankAddress to account1Addr + statediff.Args{ + OldStateRoot: block0.Root(), + NewStateRoot: block1.Root(), + BlockNumber: block1.Number(), + BlockHash: block1.Hash(), + }, + &statediff.StateObject{ + BlockNumber: block1.Number(), + BlockHash: block1.Hash(), + Nodes: []sdtypes.StateNode{ + { + Path: []byte{}, + NodeType: sdtypes.Branch, + StorageNodes: emptyStorage, + NodeValue: block1RootBranchNode, + }, + { + Path: []byte{'\x04'}, + NodeType: sdtypes.Branch, + StorageNodes: emptyStorage, + NodeValue: block1x04BranchNode, + }, + { + Path: []byte{'\x04', '\x0b'}, + NodeType: sdtypes.Branch, + StorageNodes: emptyStorage, + NodeValue: block1x040bBranchNode, + }, + { + Path: []byte{'\x04', '\x0b', '\x0e'}, + NodeType: sdtypes.Leaf, + LeafKey: block1CoinbaseHash.Bytes(), + NodeValue: block1CoinbaseLeafNode, + StorageNodes: emptyStorage, + }, + }, + }, + }, + { + "testBlock2", + // 1000 transferred from testBankAddress to account1Addr + // 1000 transferred from account1Addr to account2Addr + // account1addr creates a new contract + statediff.Args{ + OldStateRoot: block1.Root(), + NewStateRoot: block2.Root(), + BlockNumber: block2.Number(), + BlockHash: block2.Hash(), + }, + &statediff.StateObject{ + BlockNumber: block2.Number(), + BlockHash: block2.Hash(), + Nodes: []sdtypes.StateNode{ + { + Path: []byte{}, + NodeType: sdtypes.Branch, + StorageNodes: emptyStorage, + NodeValue: block2RootBranchNode, + }, + { + Path: []byte{'\x00'}, + NodeType: sdtypes.Branch, + StorageNodes: emptyStorage, + NodeValue: block2x00BranchNode, + }, + { + Path: []byte{'\x00', '\x08'}, + NodeType: sdtypes.Branch, + StorageNodes: emptyStorage, + NodeValue: block2x0008BranchNode, + }, + { + Path: []byte{'\x00', '\x08', '\x0d'}, + NodeType: sdtypes.Branch, + StorageNodes: emptyStorage, + NodeValue: block2x00080dBranchNode, + }, + // this new leaf at x00 x08 x0d x00 was "created" when a premine account (leaf) was moved from path x00 x08 x0d + // this occurred because of the creation of the new coinbase receiving account (leaf) at x00 x08 x0d x04 + // which necessitates we create a branch at x00 x08 x0d (as shown in the below UpdateAccounts) + { + Path: []byte{'\x00', '\x08', '\x0d', '\x00'}, + NodeType: sdtypes.Leaf, + StorageNodes: emptyStorage, + LeafKey: common.HexToHash("08d0f2e24db7943eab4415f99e109698863b0fecca1cf9ffc500f38cefbbe29e").Bytes(), + NodeValue: block2MovedPremineLeafNode, + }, + { + Path: []byte{'\x00', '\x08', '\x0d', '\x04'}, + NodeType: sdtypes.Leaf, + StorageNodes: emptyStorage, + LeafKey: block2CoinbaseHash.Bytes(), + NodeValue: block2CoinbaseLeafNode, + }, + }, + }, + }, + { + "testBlock3", + //the contract's storage is changed + //and the block is mined by account 2 + statediff.Args{ + OldStateRoot: block2.Root(), + NewStateRoot: block3.Root(), + BlockNumber: block3.Number(), + BlockHash: block3.Hash(), + }, + &statediff.StateObject{ + BlockNumber: block3.Number(), + BlockHash: block3.Hash(), + Nodes: []sdtypes.StateNode{ + { + Path: []byte{}, + NodeType: sdtypes.Branch, + StorageNodes: emptyStorage, + NodeValue: block3RootBranchNode, + }, + { + Path: []byte{'\x06'}, + NodeType: sdtypes.Branch, + StorageNodes: emptyStorage, + NodeValue: block3x06BranchNode, + }, + { + Path: []byte{'\x06', '\x0e'}, + NodeType: sdtypes.Branch, + StorageNodes: emptyStorage, + NodeValue: block3x060eBranchNode, + }, + { + Path: []byte{'\x0c'}, + NodeType: sdtypes.Branch, + StorageNodes: emptyStorage, + NodeValue: block3x0cBranchNode, + }, + { + Path: []byte{'\x0c', '\x0e'}, + NodeType: sdtypes.Branch, + StorageNodes: emptyStorage, + NodeValue: block3x0c0eBranchNode, + }, + { + Path: []byte{'\x0c', '\x0e', '\x05'}, + NodeType: sdtypes.Branch, + StorageNodes: emptyStorage, + NodeValue: block3x0c0e05BranchNode, + }, + { + Path: []byte{'\x0c', '\x0e', '\x05', '\x07'}, + NodeType: sdtypes.Branch, + StorageNodes: emptyStorage, + NodeValue: block3x0c0e0507BranchNode, + }, + { // How was this account created??? + Path: []byte{'\x0c', '\x0e', '\x05', '\x07', '\x03'}, + NodeType: sdtypes.Leaf, + StorageNodes: emptyStorage, + LeafKey: common.HexToHash("ce573ced93917e658d10e2d9009470dad72b63c898d173721194a12f2ae5e190").Bytes(), + NodeValue: block3MovedPremineLeafNode1, + }, + { // This account (leaf) used to be at 0c 0e 05 07, likely moves because of the new account above + Path: []byte{'\x0c', '\x0e', '\x05', '\x07', '\x08'}, + NodeType: sdtypes.Leaf, + StorageNodes: emptyStorage, + LeafKey: common.HexToHash("ce5783bc1e69eedf90f402e11f6862da14ed8e50156635a04d6393bbae154012").Bytes(), + NodeValue: block3MovedPremineLeafNode2, + }, + { // this is the new account created due to the coinbase mining a block, it's creation shouldn't affect 0x 0e 05 07 + Path: []byte{'\x06', '\x0e', '\x0f'}, + NodeType: sdtypes.Leaf, + StorageNodes: emptyStorage, + LeafKey: block3CoinbaseHash.Bytes(), + NodeValue: block3CoinbaseLeafNode, + }, + }, + }, + }, + } + + for _, test := range tests { + diff, err := builder.BuildStateDiffObject(test.startingArguments, params) + if err != nil { + t.Error(err) + } + receivedStateDiffRlp, err := rlp.EncodeToBytes(diff) + if err != nil { + t.Error(err) + } + expectedStateDiffRlp, err := rlp.EncodeToBytes(test.expected) + if err != nil { + t.Error(err) + } + sort.Slice(receivedStateDiffRlp, func(i, j int) bool { return receivedStateDiffRlp[i] < receivedStateDiffRlp[j] }) + sort.Slice(expectedStateDiffRlp, func(i, j int) bool { return expectedStateDiffRlp[i] < expectedStateDiffRlp[j] }) + if !bytes.Equal(receivedStateDiffRlp, expectedStateDiffRlp) { + t.Logf("Test failed: %s", test.name) + t.Errorf("actual state diff: %+v\nexpected state diff: %+v", diff, test.expected) + } + } +} diff --git a/statediff/metrics.go b/statediff/metrics.go new file mode 100644 index 000000000..7e7d6e328 --- /dev/null +++ b/statediff/metrics.go @@ -0,0 +1,54 @@ +package statediff + +import ( + "strings" + + "github.com/ethereum/go-ethereum/metrics" +) + +const ( + namespace = "statediff" +) + +// Build a fully qualified metric name +func metricName(subsystem, name string) string { + if name == "" { + return "" + } + parts := []string{namespace, name} + if subsystem != "" { + parts = []string{namespace, subsystem, name} + } + // Prometheus uses _ but geth metrics uses / and replaces + return strings.Join(parts, "/") +} + +type statediffMetricsHandles struct { + // Height of latest synced by core.BlockChain + // FIXME + lastSyncHeight metrics.Gauge + // Height of the latest block received from chainEvent channel + lastEventHeight metrics.Gauge + // Height of latest state diff + lastStatediffHeight metrics.Gauge + // Current length of chainEvent channels + serviceLoopChannelLen metrics.Gauge + writeLoopChannelLen metrics.Gauge +} + +func RegisterStatediffMetrics(reg metrics.Registry) statediffMetricsHandles { + ctx := statediffMetricsHandles{ + lastSyncHeight: metrics.NewGauge(), + lastEventHeight: metrics.NewGauge(), + lastStatediffHeight: metrics.NewGauge(), + serviceLoopChannelLen: metrics.NewGauge(), + writeLoopChannelLen: metrics.NewGauge(), + } + subsys := "service" + reg.Register(metricName(subsys, "last_sync_height"), ctx.lastSyncHeight) + reg.Register(metricName(subsys, "last_event_height"), ctx.lastEventHeight) + reg.Register(metricName(subsys, "last_statediff_height"), ctx.lastStatediffHeight) + reg.Register(metricName(subsys, "service_loop_channel_len"), ctx.serviceLoopChannelLen) + reg.Register(metricName(subsys, "write_loop_channel_len"), ctx.writeLoopChannelLen) + return ctx +} diff --git a/statediff/service.go b/statediff/service.go new file mode 100644 index 000000000..2f16fc4ab --- /dev/null +++ b/statediff/service.go @@ -0,0 +1,627 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package statediff + +import ( + "bytes" + "math/big" + "strconv" + "sync" + "sync/atomic" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/trie" + + ind "github.com/ethereum/go-ethereum/statediff/indexer" + nodeinfo "github.com/ethereum/go-ethereum/statediff/indexer/node" + "github.com/ethereum/go-ethereum/statediff/indexer/postgres" + . "github.com/ethereum/go-ethereum/statediff/types" +) + +const chainEventChanSize = 20000 + +var writeLoopParams = Params{ + IntermediateStateNodes: true, + IntermediateStorageNodes: true, + IncludeBlock: true, + IncludeReceipts: true, + IncludeTD: true, + IncludeCode: true, +} + +var statediffMetrics = RegisterStatediffMetrics(metrics.DefaultRegistry) + +type blockChain interface { + SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription + GetBlockByHash(hash common.Hash) *types.Block + GetBlockByNumber(number uint64) *types.Block + GetReceiptsByHash(hash common.Hash) types.Receipts + GetTdByHash(hash common.Hash) *big.Int + UnlockTrie(root common.Hash) + StateCache() state.Database +} + +// IService is the state-diffing service interface +type IService interface { + // Start() and Stop() + node.Lifecycle + // Method to getting API(s) for this service + APIs() []rpc.API + // Main event loop for processing state diffs + Loop(chainEventCh chan core.ChainEvent) + // Method to subscribe to receive state diff processing output + Subscribe(id rpc.ID, sub chan<- Payload, quitChan chan<- bool, params Params) + // Method to unsubscribe from state diff processing + Unsubscribe(id rpc.ID) error + // Method to get state diff object at specific block + StateDiffAt(blockNumber uint64, params Params) (*Payload, error) + // Method to get state trie object at specific block + StateTrieAt(blockNumber uint64, params Params) (*Payload, error) + // Method to stream out all code and codehash pairs + StreamCodeAndCodeHash(blockNumber uint64, outChan chan<- CodeAndCodeHash, quitChan chan<- bool) + // Method to write state diff object directly to DB + WriteStateDiffAt(blockNumber uint64, params Params) error + // Event loop for progressively processing and writing diffs directly to DB + WriteLoop(chainEventCh chan core.ChainEvent) +} + +// Wraps consructor parameters +type ServiceParams struct { + DBParams *DBParams + // Whether to enable writing state diffs directly to track blochain head + EnableWriteLoop bool + // Size of the worker pool + NumWorkers uint +} + +// Service is the underlying struct for the state diffing service +type Service struct { + // Used to sync access to the Subscriptions + sync.Mutex + // Used to build the state diff objects + Builder Builder + // Used to subscribe to chain events (blocks) + BlockChain blockChain + // Used to signal shutdown of the service + QuitChan chan bool + // A mapping of rpc.IDs to their subscription channels, mapped to their subscription type (hash of the Params rlp) + Subscriptions map[common.Hash]map[rpc.ID]Subscription + // A mapping of subscription params rlp hash to the corresponding subscription params + SubscriptionTypes map[common.Hash]Params + // Cache the last block so that we can avoid having to lookup the next block's parent + BlockCache blockCache + // Whether or not we have any subscribers; only if we do, do we processes state diffs + subscribers int32 + // Interface for publishing statediffs as PG-IPLD objects + indexer ind.Indexer + // Whether to enable writing state diffs directly to track blochain head + enableWriteLoop bool + // Size of the worker pool + numWorkers uint +} + +// Wrap the cached last block for safe access from different service loops +type blockCache struct { + sync.Mutex + blocks map[common.Hash]*types.Block + maxSize uint +} + +func NewBlockCache(max uint) blockCache { + return blockCache{ + blocks: make(map[common.Hash]*types.Block), + maxSize: max, + } +} + +// New creates a new statediff.Service +// func New(stack *node.Node, ethServ *eth.Ethereum, dbParams *DBParams, enableWriteLoop bool) error { +func New(stack *node.Node, ethServ *eth.Ethereum, params ServiceParams) error { + blockChain := ethServ.BlockChain() + var indexer ind.Indexer + if params.DBParams != nil { + info := nodeinfo.Info{ + GenesisBlock: blockChain.Genesis().Hash().Hex(), + NetworkID: strconv.FormatUint(ethServ.NetVersion(), 10), + ChainID: blockChain.Config().ChainID.Uint64(), + ID: params.DBParams.ID, + ClientName: params.DBParams.ClientName, + } + + // TODO: pass max idle, open, lifetime? + db, err := postgres.NewDB(params.DBParams.ConnectionURL, postgres.ConnectionConfig{}, info) + if err != nil { + return err + } + indexer = ind.NewStateDiffIndexer(blockChain.Config(), db) + } + workers := params.NumWorkers + if workers == 0 { + workers = 1 + } + sds := &Service{ + Mutex: sync.Mutex{}, + BlockChain: blockChain, + Builder: NewBuilder(blockChain.StateCache()), + QuitChan: make(chan bool), + Subscriptions: make(map[common.Hash]map[rpc.ID]Subscription), + SubscriptionTypes: make(map[common.Hash]Params), + BlockCache: NewBlockCache(workers), + indexer: indexer, + enableWriteLoop: params.EnableWriteLoop, + numWorkers: workers, + } + stack.RegisterLifecycle(sds) + stack.RegisterAPIs(sds.APIs()) + return nil +} + +// Protocols exports the services p2p protocols, this service has none +func (sds *Service) Protocols() []p2p.Protocol { + return []p2p.Protocol{} +} + +// APIs returns the RPC descriptors the statediff.Service offers +func (sds *Service) APIs() []rpc.API { + return []rpc.API{ + { + Namespace: APIName, + Version: APIVersion, + Service: NewPublicStateDiffAPI(sds), + Public: true, + }, + } +} + +// Return the parent block of currentBlock, using the cached block if available; +// and cache the passed block +func (lbc *blockCache) getParentBlock(currentBlock *types.Block, bc blockChain) *types.Block { + lbc.Lock() + parentHash := currentBlock.ParentHash() + var parentBlock *types.Block + if block, ok := lbc.blocks[parentHash]; ok { + parentBlock = block + if len(lbc.blocks) > int(lbc.maxSize) { + delete(lbc.blocks, parentHash) + } + } else { + parentBlock = bc.GetBlockByHash(parentHash) + } + lbc.blocks[currentBlock.Hash()] = currentBlock + lbc.Unlock() + return parentBlock +} + +type workerParams struct { + chainEventCh <-chan core.ChainEvent + errCh <-chan error + wg *sync.WaitGroup + id uint +} + +func (sds *Service) WriteLoop(chainEventCh chan core.ChainEvent) { + chainEventSub := sds.BlockChain.SubscribeChainEvent(chainEventCh) + defer chainEventSub.Unsubscribe() + errCh := chainEventSub.Err() + var wg sync.WaitGroup + // Process metrics for chain events, then forward to workers + chainEventFwd := make(chan core.ChainEvent, chainEventChanSize) + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case chainEvent := <-chainEventCh: + statediffMetrics.lastEventHeight.Update(int64(chainEvent.Block.Number().Uint64())) + statediffMetrics.writeLoopChannelLen.Update(int64(len(chainEventCh))) + chainEventFwd <- chainEvent + case <-sds.QuitChan: + return + } + } + }() + wg.Add(int(sds.numWorkers)) + for worker := uint(0); worker < sds.numWorkers; worker++ { + params := workerParams{chainEventCh: chainEventFwd, errCh: errCh, wg: &wg, id: worker} + go sds.writeLoopWorker(params) + } + wg.Wait() +} + +func (sds *Service) writeLoopWorker(params workerParams) { + defer params.wg.Done() + for { + select { + //Notify chain event channel of events + case chainEvent := <-params.chainEventCh: + log.Debug("WriteLoop(): chain event received", "event", chainEvent) + currentBlock := chainEvent.Block + parentBlock := sds.BlockCache.getParentBlock(currentBlock, sds.BlockChain) + if parentBlock == nil { + log.Error("Parent block is nil, skipping this block", "block height", currentBlock.Number()) + continue + } + log.Info("Writing state diff", "block height", currentBlock.Number().Uint64(), "worker", params.id) + err := sds.writeStateDiff(currentBlock, parentBlock.Root(), writeLoopParams) + if err != nil { + log.Error("statediff.Service.WriteLoop: processing error", "block height", currentBlock.Number().Uint64(), "error", err.Error(), "worker", params.id) + continue + } + // TODO: how to handle with concurrent workers + statediffMetrics.lastStatediffHeight.Update(int64(currentBlock.Number().Uint64())) + case err := <-params.errCh: + log.Warn("Error from chain event subscription", "error", err, "worker", params.id) + sds.close() + return + case <-sds.QuitChan: + log.Info("Quitting the statediff writing process", "worker", params.id) + sds.close() + return + } + } +} + +// Loop is the main processing method +func (sds *Service) Loop(chainEventCh chan core.ChainEvent) { + chainEventSub := sds.BlockChain.SubscribeChainEvent(chainEventCh) + defer chainEventSub.Unsubscribe() + errCh := chainEventSub.Err() + for { + select { + //Notify chain event channel of events + case chainEvent := <-chainEventCh: + statediffMetrics.serviceLoopChannelLen.Update(int64(len(chainEventCh))) + log.Debug("Loop(): chain event received", "event", chainEvent) + // if we don't have any subscribers, do not process a statediff + if atomic.LoadInt32(&sds.subscribers) == 0 { + log.Debug("Currently no subscribers to the statediffing service; processing is halted") + continue + } + currentBlock := chainEvent.Block + parentBlock := sds.BlockCache.getParentBlock(currentBlock, sds.BlockChain) + if parentBlock == nil { + log.Error("Parent block is nil, skipping this block", "block height", currentBlock.Number()) + continue + } + sds.streamStateDiff(currentBlock, parentBlock.Root()) + case err := <-errCh: + log.Warn("Error from chain event subscription", "error", err) + sds.close() + return + case <-sds.QuitChan: + log.Info("Quitting the statediffing process") + sds.close() + return + } + } +} + +// streamStateDiff method builds the state diff payload for each subscription according to their subscription type and sends them the result +func (sds *Service) streamStateDiff(currentBlock *types.Block, parentRoot common.Hash) { + sds.Lock() + for ty, subs := range sds.Subscriptions { + params, ok := sds.SubscriptionTypes[ty] + if !ok { + log.Error("no parameter set associated with this subscription", "subscription type", ty.Hex()) + sds.closeType(ty) + continue + } + // create payload for this subscription type + payload, err := sds.processStateDiff(currentBlock, parentRoot, params) + if err != nil { + log.Error("statediff processing error", "block height", currentBlock.Number().Uint64(), "parameters", params, "error", err.Error()) + continue + } + for id, sub := range subs { + select { + case sub.PayloadChan <- *payload: + log.Debug("sending statediff payload at head", "height", currentBlock.Number(), "subscription id", id) + default: + log.Info("unable to send statediff payload; channel has no receiver", "subscription id", id) + } + } + } + sds.Unlock() +} + +// StateDiffAt returns a state diff object payload at the specific blockheight +// This operation cannot be performed back past the point of db pruning; it requires an archival node for historical data +func (sds *Service) StateDiffAt(blockNumber uint64, params Params) (*Payload, error) { + currentBlock := sds.BlockChain.GetBlockByNumber(blockNumber) + log.Info("sending state diff", "block height", blockNumber) + if blockNumber == 0 { + return sds.processStateDiff(currentBlock, common.Hash{}, params) + } + parentBlock := sds.BlockChain.GetBlockByHash(currentBlock.ParentHash()) + return sds.processStateDiff(currentBlock, parentBlock.Root(), params) +} + +// processStateDiff method builds the state diff payload from the current block, parent state root, and provided params +func (sds *Service) processStateDiff(currentBlock *types.Block, parentRoot common.Hash, params Params) (*Payload, error) { + stateDiff, err := sds.Builder.BuildStateDiffObject(Args{ + NewStateRoot: currentBlock.Root(), + OldStateRoot: parentRoot, + BlockHash: currentBlock.Hash(), + BlockNumber: currentBlock.Number(), + }, params) + // allow dereferencing of parent, keep current locked as it should be the next parent + sds.BlockChain.UnlockTrie(parentRoot) + if err != nil { + return nil, err + } + stateDiffRlp, err := rlp.EncodeToBytes(stateDiff) + if err != nil { + return nil, err + } + log.Info("state diff size", "at block height", currentBlock.Number().Uint64(), "rlp byte size", len(stateDiffRlp)) + return sds.newPayload(stateDiffRlp, currentBlock, params) +} + +func (sds *Service) newPayload(stateObject []byte, block *types.Block, params Params) (*Payload, error) { + payload := &Payload{ + StateObjectRlp: stateObject, + } + if params.IncludeBlock { + blockBuff := new(bytes.Buffer) + if err := block.EncodeRLP(blockBuff); err != nil { + return nil, err + } + payload.BlockRlp = blockBuff.Bytes() + } + if params.IncludeTD { + payload.TotalDifficulty = sds.BlockChain.GetTdByHash(block.Hash()) + } + if params.IncludeReceipts { + receiptBuff := new(bytes.Buffer) + receipts := sds.BlockChain.GetReceiptsByHash(block.Hash()) + if err := rlp.Encode(receiptBuff, receipts); err != nil { + return nil, err + } + payload.ReceiptsRlp = receiptBuff.Bytes() + } + return payload, nil +} + +// StateTrieAt returns a state trie object payload at the specified blockheight +// This operation cannot be performed back past the point of db pruning; it requires an archival node for historical data +func (sds *Service) StateTrieAt(blockNumber uint64, params Params) (*Payload, error) { + currentBlock := sds.BlockChain.GetBlockByNumber(blockNumber) + log.Info("sending state trie", "block height", blockNumber) + return sds.processStateTrie(currentBlock, params) +} + +func (sds *Service) processStateTrie(block *types.Block, params Params) (*Payload, error) { + stateNodes, err := sds.Builder.BuildStateTrieObject(block) + if err != nil { + return nil, err + } + stateTrieRlp, err := rlp.EncodeToBytes(stateNodes) + if err != nil { + return nil, err + } + log.Info("state trie size", "at block height", block.Number().Uint64(), "rlp byte size", len(stateTrieRlp)) + return sds.newPayload(stateTrieRlp, block, params) +} + +// Subscribe is used by the API to subscribe to the service loop +func (sds *Service) Subscribe(id rpc.ID, sub chan<- Payload, quitChan chan<- bool, params Params) { + log.Info("Subscribing to the statediff service") + if atomic.CompareAndSwapInt32(&sds.subscribers, 0, 1) { + log.Info("State diffing subscription received; beginning statediff processing") + } + // Subscription type is defined as the hash of the rlp-serialized subscription params + by, err := rlp.EncodeToBytes(params) + if err != nil { + log.Error("State diffing params need to be rlp-serializable") + return + } + subscriptionType := crypto.Keccak256Hash(by) + // Add subscriber + sds.Lock() + if sds.Subscriptions[subscriptionType] == nil { + sds.Subscriptions[subscriptionType] = make(map[rpc.ID]Subscription) + } + sds.Subscriptions[subscriptionType][id] = Subscription{ + PayloadChan: sub, + QuitChan: quitChan, + } + sds.SubscriptionTypes[subscriptionType] = params + sds.Unlock() +} + +// Unsubscribe is used to unsubscribe from the service loop +func (sds *Service) Unsubscribe(id rpc.ID) error { + log.Info("Unsubscribing from the statediff service", "subscription id", id) + sds.Lock() + for ty := range sds.Subscriptions { + delete(sds.Subscriptions[ty], id) + if len(sds.Subscriptions[ty]) == 0 { + // If we removed the last subscription of this type, remove the subscription type outright + delete(sds.Subscriptions, ty) + delete(sds.SubscriptionTypes, ty) + } + } + if len(sds.Subscriptions) == 0 { + if atomic.CompareAndSwapInt32(&sds.subscribers, 1, 0) { + log.Info("No more subscriptions; halting statediff processing") + } + } + sds.Unlock() + return nil +} + +// Start is used to begin the service +func (sds *Service) Start() error { + log.Info("Starting statediff service") + + chainEventCh := make(chan core.ChainEvent, chainEventChanSize) + go sds.Loop(chainEventCh) + + if sds.enableWriteLoop { + log.Info("Starting statediff DB write loop", "params", writeLoopParams) + chainEventCh := make(chan core.ChainEvent, chainEventChanSize) + go sds.WriteLoop(chainEventCh) + } + + return nil +} + +// Stop is used to close down the service +func (sds *Service) Stop() error { + log.Info("Stopping statediff service") + close(sds.QuitChan) + return nil +} + +// close is used to close all listening subscriptions +func (sds *Service) close() { + sds.Lock() + for ty, subs := range sds.Subscriptions { + for id, sub := range subs { + select { + case sub.QuitChan <- true: + log.Info("closing subscription", "id", id) + default: + log.Info("unable to close subscription; channel has no receiver", "subscription id", id) + } + delete(sds.Subscriptions[ty], id) + } + delete(sds.Subscriptions, ty) + delete(sds.SubscriptionTypes, ty) + } + sds.Unlock() +} + +// closeType is used to close all subscriptions of given type +// closeType needs to be called with subscription access locked +func (sds *Service) closeType(subType common.Hash) { + subs := sds.Subscriptions[subType] + for id, sub := range subs { + sendNonBlockingQuit(id, sub) + } + delete(sds.Subscriptions, subType) + delete(sds.SubscriptionTypes, subType) +} + +func sendNonBlockingQuit(id rpc.ID, sub Subscription) { + select { + case sub.QuitChan <- true: + log.Info("closing subscription", "id", id) + default: + log.Info("unable to close subscription; channel has no receiver", "subscription id", id) + } +} + +// StreamCodeAndCodeHash subscription method for extracting all the codehash=>code mappings that exist in the trie at the provided height +func (sds *Service) StreamCodeAndCodeHash(blockNumber uint64, outChan chan<- CodeAndCodeHash, quitChan chan<- bool) { + current := sds.BlockChain.GetBlockByNumber(blockNumber) + log.Info("sending code and codehash", "block height", blockNumber) + currentTrie, err := sds.BlockChain.StateCache().OpenTrie(current.Root()) + if err != nil { + log.Error("error creating trie for block", "block height", current.Number(), "err", err) + close(quitChan) + return + } + it := currentTrie.NodeIterator([]byte{}) + leafIt := trie.NewIterator(it) + go func() { + defer close(quitChan) + for leafIt.Next() { + select { + case <-sds.QuitChan: + return + default: + } + account := new(state.Account) + if err := rlp.DecodeBytes(leafIt.Value, account); err != nil { + log.Error("error decoding state account", "err", err) + return + } + codeHash := common.BytesToHash(account.CodeHash) + code, err := sds.BlockChain.StateCache().ContractCode(common.Hash{}, codeHash) + if err != nil { + log.Error("error collecting contract code", "err", err) + return + } + outChan <- CodeAndCodeHash{ + Hash: codeHash, + Code: code, + } + } + }() +} + +// WriteStateDiffAt writes a state diff at the specific blockheight directly to the database +// This operation cannot be performed back past the point of db pruning; it requires an archival node +// for historical data +func (sds *Service) WriteStateDiffAt(blockNumber uint64, params Params) error { + currentBlock := sds.BlockChain.GetBlockByNumber(blockNumber) + parentRoot := common.Hash{} + if blockNumber != 0 { + parentBlock := sds.BlockChain.GetBlockByHash(currentBlock.ParentHash()) + parentRoot = parentBlock.Root() + } + return sds.writeStateDiff(currentBlock, parentRoot, params) +} + +// Writes a state diff from the current block, parent state root, and provided params +func (sds *Service) writeStateDiff(block *types.Block, parentRoot common.Hash, params Params) error { + // log.Info("Writing state diff", "block height", block.Number().Uint64()) + var totalDifficulty *big.Int + var receipts types.Receipts + if params.IncludeTD { + totalDifficulty = sds.BlockChain.GetTdByHash(block.Hash()) + } + if params.IncludeReceipts { + receipts = sds.BlockChain.GetReceiptsByHash(block.Hash()) + } + tx, err := sds.indexer.PushBlock(block, receipts, totalDifficulty) + if err != nil { + return err + } + // defer handling of commit/rollback for any return case + defer tx.Close() + output := func(node StateNode) error { + return sds.indexer.PushStateNode(tx, node) + } + codeOutput := func(c CodeAndCodeHash) error { + return sds.indexer.PushCodeAndCodeHash(tx, c) + } + err = sds.Builder.WriteStateDiffObject(StateRoots{ + NewStateRoot: block.Root(), + OldStateRoot: parentRoot, + }, params, output, codeOutput) + + // allow dereferencing of parent, keep current locked as it should be the next parent + sds.BlockChain.UnlockTrie(parentRoot) + if err != nil { + return err + } + return nil +} diff --git a/statediff/service_test.go b/statediff/service_test.go new file mode 100644 index 000000000..ca9a483a5 --- /dev/null +++ b/statediff/service_test.go @@ -0,0 +1,291 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package statediff_test + +import ( + "bytes" + "math/big" + "math/rand" + "reflect" + "sync" + "testing" + + "github.com/ethereum/go-ethereum/trie" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/rpc" + statediff "github.com/ethereum/go-ethereum/statediff" + "github.com/ethereum/go-ethereum/statediff/testhelpers/mocks" +) + +func TestServiceLoop(t *testing.T) { + testErrorInChainEventLoop(t) + testErrorInBlockLoop(t) +} + +var ( + eventsChannel = make(chan core.ChainEvent, 1) + + parentRoot1 = common.HexToHash("0x01") + parentRoot2 = common.HexToHash("0x02") + parentHeader1 = types.Header{Number: big.NewInt(rand.Int63()), Root: parentRoot1} + parentHeader2 = types.Header{Number: big.NewInt(rand.Int63()), Root: parentRoot2} + + parentBlock1 = types.NewBlock(&parentHeader1, nil, nil, nil, new(trie.Trie)) + parentBlock2 = types.NewBlock(&parentHeader2, nil, nil, nil, new(trie.Trie)) + + parentHash1 = parentBlock1.Hash() + parentHash2 = parentBlock2.Hash() + + testRoot1 = common.HexToHash("0x03") + testRoot2 = common.HexToHash("0x04") + testRoot3 = common.HexToHash("0x04") + header1 = types.Header{ParentHash: parentHash1, Root: testRoot1, Number: big.NewInt(1)} + header2 = types.Header{ParentHash: parentHash2, Root: testRoot2, Number: big.NewInt(2)} + header3 = types.Header{ParentHash: common.HexToHash("parent hash"), Root: testRoot3, Number: big.NewInt(3)} + + testBlock1 = types.NewBlock(&header1, nil, nil, nil, new(trie.Trie)) + testBlock2 = types.NewBlock(&header2, nil, nil, nil, new(trie.Trie)) + testBlock3 = types.NewBlock(&header3, nil, nil, nil, new(trie.Trie)) + + receiptRoot1 = common.HexToHash("0x05") + receiptRoot2 = common.HexToHash("0x06") + receiptRoot3 = common.HexToHash("0x07") + testReceipts1 = []*types.Receipt{types.NewReceipt(receiptRoot1.Bytes(), false, 1000), types.NewReceipt(receiptRoot2.Bytes(), false, 2000)} + testReceipts2 = []*types.Receipt{types.NewReceipt(receiptRoot3.Bytes(), false, 3000)} + + event1 = core.ChainEvent{Block: testBlock1} + event2 = core.ChainEvent{Block: testBlock2} + event3 = core.ChainEvent{Block: testBlock3} + + defaultParams = statediff.Params{ + IncludeBlock: true, + IncludeReceipts: true, + IncludeTD: true, + } +) + +func testErrorInChainEventLoop(t *testing.T) { + //the first chain event causes and error (in blockchain mock) + builder := mocks.Builder{} + blockChain := mocks.BlockChain{} + serviceQuit := make(chan bool) + service := statediff.Service{ + Mutex: sync.Mutex{}, + Builder: &builder, + BlockChain: &blockChain, + QuitChan: serviceQuit, + Subscriptions: make(map[common.Hash]map[rpc.ID]statediff.Subscription), + SubscriptionTypes: make(map[common.Hash]statediff.Params), + BlockCache: statediff.NewBlockCache(1), + } + payloadChan := make(chan statediff.Payload, 2) + quitChan := make(chan bool) + service.Subscribe(rpc.NewID(), payloadChan, quitChan, defaultParams) + testRoot2 = common.HexToHash("0xTestRoot2") + blockMapping := make(map[common.Hash]*types.Block) + blockMapping[parentBlock1.Hash()] = parentBlock1 + blockMapping[parentBlock2.Hash()] = parentBlock2 + blockChain.SetBlocksForHashes(blockMapping) + blockChain.SetChainEvents([]core.ChainEvent{event1, event2, event3}) + blockChain.SetReceiptsForHash(testBlock1.Hash(), testReceipts1) + blockChain.SetReceiptsForHash(testBlock2.Hash(), testReceipts2) + + payloads := make([]statediff.Payload, 0, 2) + wg := new(sync.WaitGroup) + go func() { + wg.Add(1) + for i := 0; i < 2; i++ { + select { + case payload := <-payloadChan: + payloads = append(payloads, payload) + case <-quitChan: + } + } + wg.Done() + }() + service.Loop(eventsChannel) + wg.Wait() + if len(payloads) != 2 { + t.Error("Test failure:", t.Name()) + t.Logf("Actual number of payloads does not equal expected.\nactual: %+v\nexpected: 3", len(payloads)) + } + + testReceipts1Rlp, err := rlp.EncodeToBytes(testReceipts1) + if err != nil { + t.Error(err) + } + testReceipts2Rlp, err := rlp.EncodeToBytes(testReceipts2) + if err != nil { + t.Error(err) + } + expectedReceiptsRlp := [][]byte{testReceipts1Rlp, testReceipts2Rlp, nil} + for i, payload := range payloads { + if !bytes.Equal(payload.ReceiptsRlp, expectedReceiptsRlp[i]) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual receipt rlp for payload %d does not equal expected.\nactual: %+v\nexpected: %+v", i, payload.ReceiptsRlp, expectedReceiptsRlp[i]) + } + } + + if !reflect.DeepEqual(builder.Params, defaultParams) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual params does not equal expected.\nactual:%+v\nexpected: %+v", builder.Params, defaultParams) + } + if !bytes.Equal(builder.Args.BlockHash.Bytes(), testBlock2.Hash().Bytes()) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual blockhash does not equal expected.\nactual:%x\nexpected: %x", builder.Args.BlockHash.Bytes(), testBlock2.Hash().Bytes()) + } + if !bytes.Equal(builder.Args.OldStateRoot.Bytes(), parentBlock2.Root().Bytes()) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual root does not equal expected.\nactual:%x\nexpected: %x", builder.Args.OldStateRoot.Bytes(), parentBlock2.Root().Bytes()) + } + if !bytes.Equal(builder.Args.NewStateRoot.Bytes(), testBlock2.Root().Bytes()) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual root does not equal expected.\nactual:%x\nexpected: %x", builder.Args.NewStateRoot.Bytes(), testBlock2.Root().Bytes()) + } + //look up the parent block from its hash + expectedHashes := []common.Hash{testBlock1.ParentHash(), testBlock2.ParentHash()} + if !reflect.DeepEqual(blockChain.HashesLookedUp, expectedHashes) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual looked up parent hashes does not equal expected.\nactual:%+v\nexpected: %+v", blockChain.HashesLookedUp, expectedHashes) + } +} + +func testErrorInBlockLoop(t *testing.T) { + //second block's parent block can't be found + builder := mocks.Builder{} + blockChain := mocks.BlockChain{} + service := statediff.Service{ + Builder: &builder, + BlockChain: &blockChain, + QuitChan: make(chan bool), + Subscriptions: make(map[common.Hash]map[rpc.ID]statediff.Subscription), + SubscriptionTypes: make(map[common.Hash]statediff.Params), + BlockCache: statediff.NewBlockCache(1), + } + payloadChan := make(chan statediff.Payload) + quitChan := make(chan bool) + service.Subscribe(rpc.NewID(), payloadChan, quitChan, defaultParams) + blockMapping := make(map[common.Hash]*types.Block) + blockMapping[parentBlock1.Hash()] = parentBlock1 + blockChain.SetBlocksForHashes(blockMapping) + blockChain.SetChainEvents([]core.ChainEvent{event1, event2}) + // Need to have listeners on the channels or the subscription will be closed and the processing halted + go func() { + select { + case <-payloadChan: + case <-quitChan: + } + }() + service.Loop(eventsChannel) + if !reflect.DeepEqual(builder.Params, defaultParams) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual params does not equal expected.\nactual:%+v\nexpected: %+v", builder.Params, defaultParams) + } + if !bytes.Equal(builder.Args.BlockHash.Bytes(), testBlock1.Hash().Bytes()) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual blockhash does not equal expected.\nactual:%+v\nexpected: %x", builder.Args.BlockHash.Bytes(), testBlock1.Hash().Bytes()) + } + if !bytes.Equal(builder.Args.OldStateRoot.Bytes(), parentBlock1.Root().Bytes()) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual old state root does not equal expected.\nactual:%+v\nexpected: %x", builder.Args.OldStateRoot.Bytes(), parentBlock1.Root().Bytes()) + } + if !bytes.Equal(builder.Args.NewStateRoot.Bytes(), testBlock1.Root().Bytes()) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual new state root does not equal expected.\nactual:%+v\nexpected: %x", builder.Args.NewStateRoot.Bytes(), testBlock1.Root().Bytes()) + } +} + +func TestGetStateDiffAt(t *testing.T) { + testErrorInStateDiffAt(t) +} + +func testErrorInStateDiffAt(t *testing.T) { + mockStateDiff := statediff.StateObject{ + BlockNumber: testBlock1.Number(), + BlockHash: testBlock1.Hash(), + } + expectedStateDiffRlp, err := rlp.EncodeToBytes(mockStateDiff) + if err != nil { + t.Error(err) + } + expectedReceiptsRlp, err := rlp.EncodeToBytes(testReceipts1) + if err != nil { + t.Error(err) + } + expectedBlockRlp, err := rlp.EncodeToBytes(testBlock1) + if err != nil { + t.Error(err) + } + expectedStateDiffPayload := statediff.Payload{ + StateObjectRlp: expectedStateDiffRlp, + ReceiptsRlp: expectedReceiptsRlp, + BlockRlp: expectedBlockRlp, + } + expectedStateDiffPayloadRlp, err := rlp.EncodeToBytes(expectedStateDiffPayload) + if err != nil { + t.Error(err) + } + builder := mocks.Builder{} + builder.SetStateDiffToBuild(mockStateDiff) + blockChain := mocks.BlockChain{} + blockMapping := make(map[common.Hash]*types.Block) + blockMapping[parentBlock1.Hash()] = parentBlock1 + blockChain.SetBlocksForHashes(blockMapping) + blockChain.SetBlockForNumber(testBlock1, testBlock1.NumberU64()) + blockChain.SetReceiptsForHash(testBlock1.Hash(), testReceipts1) + service := statediff.Service{ + Mutex: sync.Mutex{}, + Builder: &builder, + BlockChain: &blockChain, + QuitChan: make(chan bool), + Subscriptions: make(map[common.Hash]map[rpc.ID]statediff.Subscription), + SubscriptionTypes: make(map[common.Hash]statediff.Params), + BlockCache: statediff.NewBlockCache(1), + } + stateDiffPayload, err := service.StateDiffAt(testBlock1.NumberU64(), defaultParams) + if err != nil { + t.Error(err) + } + stateDiffPayloadRlp, err := rlp.EncodeToBytes(stateDiffPayload) + if err != nil { + t.Error(err) + } + if !reflect.DeepEqual(builder.Params, defaultParams) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual params does not equal expected.\nactual:%+v\nexpected: %+v", builder.Params, defaultParams) + } + if !bytes.Equal(builder.Args.BlockHash.Bytes(), testBlock1.Hash().Bytes()) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual blockhash does not equal expected.\nactual:%+v\nexpected: %x", builder.Args.BlockHash.Bytes(), testBlock1.Hash().Bytes()) + } + if !bytes.Equal(builder.Args.OldStateRoot.Bytes(), parentBlock1.Root().Bytes()) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual old state root does not equal expected.\nactual:%+v\nexpected: %x", builder.Args.OldStateRoot.Bytes(), parentBlock1.Root().Bytes()) + } + if !bytes.Equal(builder.Args.NewStateRoot.Bytes(), testBlock1.Root().Bytes()) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual new state root does not equal expected.\nactual:%+v\nexpected: %x", builder.Args.NewStateRoot.Bytes(), testBlock1.Root().Bytes()) + } + if !bytes.Equal(expectedStateDiffPayloadRlp, stateDiffPayloadRlp) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual state diff payload does not equal expected.\nactual:%+v\nexpected: %+v", expectedStateDiffPayload, stateDiffPayload) + } +} diff --git a/statediff/testhelpers/helpers.go b/statediff/testhelpers/helpers.go new file mode 100644 index 000000000..7fd320b25 --- /dev/null +++ b/statediff/testhelpers/helpers.go @@ -0,0 +1,124 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package testhelpers + +import ( + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/params" +) + +// MakeChain creates a chain of n blocks starting at and including parent. +// the returned hash chain is ordered head->parent. +func MakeChain(n int, parent *types.Block, chainGen func(int, *core.BlockGen)) ([]*types.Block, *core.BlockChain) { + config := params.TestChainConfig + blocks, _ := core.GenerateChain(config, parent, ethash.NewFaker(), Testdb, n, chainGen) + chain, _ := core.NewBlockChain(Testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil) + return blocks, chain +} + +func TestSelfDestructChainGen(i int, block *core.BlockGen) { + signer := types.HomesteadSigner{} + switch i { + case 0: + // Block 1 is mined by Account1Addr + // Account1Addr creates a new contract + block.SetCoinbase(TestBankAddress) + tx, _ := types.SignTx(types.NewContractCreation(0, big.NewInt(0), 1000000, big.NewInt(0), ContractCode), signer, TestBankKey) + ContractAddr = crypto.CreateAddress(TestBankAddress, 0) + block.AddTx(tx) + case 1: + // Block 2 is mined by Account1Addr + // Account1Addr self-destructs the contract + block.SetCoinbase(TestBankAddress) + data := common.Hex2Bytes("43D726D6") + tx, _ := types.SignTx(types.NewTransaction(1, ContractAddr, big.NewInt(0), 100000, nil, data), signer, TestBankKey) + block.AddTx(tx) + } +} + +func TestChainGen(i int, block *core.BlockGen) { + signer := types.HomesteadSigner{} + switch i { + case 0: + // In block 1, the test bank sends account #1 some ether. + tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), Account1Addr, big.NewInt(10000), params.TxGas, nil, nil), signer, TestBankKey) + block.AddTx(tx) + case 1: + // In block 2, the test bank sends some more ether to account #1. + // Account1Addr passes it on to account #2. + // Account1Addr creates a test contract. + tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), Account1Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, TestBankKey) + nonce := block.TxNonce(Account1Addr) + tx2, _ := types.SignTx(types.NewTransaction(nonce, Account2Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, Account1Key) + nonce++ + tx3, _ := types.SignTx(types.NewContractCreation(nonce, big.NewInt(0), 1000000, big.NewInt(0), ContractCode), signer, Account1Key) + ContractAddr = crypto.CreateAddress(Account1Addr, nonce) + block.AddTx(tx1) + block.AddTx(tx2) + block.AddTx(tx3) + case 2: + // Block 3 has a single tx from the bankAccount to the contract, that transfers no value + // Block 3 is mined by Account2Addr + block.SetCoinbase(Account2Addr) + //put function: c16431b9 + //close function: 43d726d6 + data := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003") + tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), ContractAddr, big.NewInt(0), 100000, nil, data), signer, TestBankKey) + block.AddTx(tx) + case 3: + // Block 4 has three txs from bankAccount to the contract, that transfer no value + // Two set the two original slot positions to 0 and one sets another position to a new value + // Block 4 is mined by Account2Addr + block.SetCoinbase(Account2Addr) + data1 := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000") + data2 := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000") + data3 := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000009") + + nonce := block.TxNonce(TestBankAddress) + tx1, _ := types.SignTx(types.NewTransaction(nonce, ContractAddr, big.NewInt(0), 100000, nil, data1), signer, TestBankKey) + nonce++ + tx2, _ := types.SignTx(types.NewTransaction(nonce, ContractAddr, big.NewInt(0), 100000, nil, data2), signer, TestBankKey) + nonce++ + tx3, _ := types.SignTx(types.NewTransaction(nonce, ContractAddr, big.NewInt(0), 100000, nil, data3), signer, TestBankKey) + block.AddTx(tx1) + block.AddTx(tx2) + block.AddTx(tx3) + case 4: + // Block 5 has one tx from bankAccount to the contract, that transfers no value + // It sets the remaining storage value to zero + // Block 5 is mined by Account1Addr + block.SetCoinbase(Account1Addr) + data := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000") + nonce := block.TxNonce(TestBankAddress) + tx, _ := types.SignTx(types.NewTransaction(nonce, ContractAddr, big.NewInt(0), 100000, nil, data), signer, TestBankKey) + block.AddTx(tx) + case 5: + // Block 6 has a tx from Account1Key which self-destructs the contract, it transfers no value + // Block 6 is mined by Account2Addr + block.SetCoinbase(Account2Addr) + data := common.Hex2Bytes("43D726D6") + tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(Account1Addr), ContractAddr, big.NewInt(0), 100000, nil, data), signer, Account1Key) + block.AddTx(tx) + } +} diff --git a/statediff/testhelpers/mocks/blockchain.go b/statediff/testhelpers/mocks/blockchain.go new file mode 100644 index 000000000..b0111e64c --- /dev/null +++ b/statediff/testhelpers/mocks/blockchain.go @@ -0,0 +1,134 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package mocks + +import ( + "errors" + "math/big" + "time" + + "github.com/ethereum/go-ethereum/core/state" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/event" +) + +// BlockChain is a mock blockchain for testing +type BlockChain struct { + HashesLookedUp []common.Hash + blocksToReturnByHash map[common.Hash]*types.Block + blocksToReturnByNumber map[uint64]*types.Block + callCount int + ChainEvents []core.ChainEvent + Receipts map[common.Hash]types.Receipts + TDByHash map[common.Hash]*big.Int +} + +// SetBlocksForHashes mock method +func (blockChain *BlockChain) SetBlocksForHashes(blocks map[common.Hash]*types.Block) { + if blockChain.blocksToReturnByHash == nil { + blockChain.blocksToReturnByHash = make(map[common.Hash]*types.Block) + } + blockChain.blocksToReturnByHash = blocks +} + +// GetBlockByHash mock method +func (blockChain *BlockChain) GetBlockByHash(hash common.Hash) *types.Block { + blockChain.HashesLookedUp = append(blockChain.HashesLookedUp, hash) + + var block *types.Block + if len(blockChain.blocksToReturnByHash) > 0 { + block = blockChain.blocksToReturnByHash[hash] + } + + return block +} + +// SetChainEvents mock method +func (blockChain *BlockChain) SetChainEvents(chainEvents []core.ChainEvent) { + blockChain.ChainEvents = chainEvents +} + +// SubscribeChainEvent mock method +func (blockChain *BlockChain) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription { + subErr := errors.New("subscription error") + + var eventCounter int + subscription := event.NewSubscription(func(quit <-chan struct{}) error { + for _, chainEvent := range blockChain.ChainEvents { + if eventCounter > 1 { + time.Sleep(250 * time.Millisecond) + return subErr + } + select { + case ch <- chainEvent: + case <-quit: + return nil + } + eventCounter++ + } + return nil + }) + + return subscription +} + +// SetReceiptsForHash test method +func (blockChain *BlockChain) SetReceiptsForHash(hash common.Hash, receipts types.Receipts) { + if blockChain.Receipts == nil { + blockChain.Receipts = make(map[common.Hash]types.Receipts) + } + blockChain.Receipts[hash] = receipts +} + +// GetReceiptsByHash mock method +func (blockChain *BlockChain) GetReceiptsByHash(hash common.Hash) types.Receipts { + return blockChain.Receipts[hash] +} + +// SetBlockForNumber test method +func (blockChain *BlockChain) SetBlockForNumber(block *types.Block, number uint64) { + if blockChain.blocksToReturnByNumber == nil { + blockChain.blocksToReturnByNumber = make(map[uint64]*types.Block) + } + blockChain.blocksToReturnByNumber[number] = block +} + +// GetBlockByNumber mock method +func (blockChain *BlockChain) GetBlockByNumber(number uint64) *types.Block { + return blockChain.blocksToReturnByNumber[number] +} + +// GetTdByHash mock method +func (blockChain *BlockChain) GetTdByHash(hash common.Hash) *big.Int { + return blockChain.TDByHash[hash] +} + +func (blockChain *BlockChain) SetTdByHash(hash common.Hash, td *big.Int) { + if blockChain.TDByHash == nil { + blockChain.TDByHash = make(map[common.Hash]*big.Int) + } + blockChain.TDByHash[hash] = td +} + +func (blockChain *BlockChain) UnlockTrie(root common.Hash) {} + +func (BlockChain *BlockChain) StateCache() state.Database { + return nil +} diff --git a/statediff/testhelpers/mocks/builder.go b/statediff/testhelpers/mocks/builder.go new file mode 100644 index 000000000..ff9faf3ec --- /dev/null +++ b/statediff/testhelpers/mocks/builder.go @@ -0,0 +1,67 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package mocks + +import ( + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/statediff" + sdtypes "github.com/ethereum/go-ethereum/statediff/types" +) + +// Builder is a mock state diff builder +type Builder struct { + Args statediff.Args + Params statediff.Params + StateRoots statediff.StateRoots + stateDiff statediff.StateObject + block *types.Block + stateTrie statediff.StateObject + builderError error +} + +// BuildStateDiffObject mock method +func (builder *Builder) BuildStateDiffObject(args statediff.Args, params statediff.Params) (statediff.StateObject, error) { + builder.Args = args + builder.Params = params + + return builder.stateDiff, builder.builderError +} + +// BuildStateDiffObject mock method +func (builder *Builder) WriteStateDiffObject(args statediff.StateRoots, params statediff.Params, output sdtypes.StateNodeSink, codeOutput sdtypes.CodeSink) error { + builder.StateRoots = args + builder.Params = params + + return builder.builderError +} + +// BuildStateTrieObject mock method +func (builder *Builder) BuildStateTrieObject(block *types.Block) (statediff.StateObject, error) { + builder.block = block + + return builder.stateTrie, builder.builderError +} + +// SetStateDiffToBuild mock method +func (builder *Builder) SetStateDiffToBuild(stateDiff statediff.StateObject) { + builder.stateDiff = stateDiff +} + +// SetBuilderError mock method +func (builder *Builder) SetBuilderError(err error) { + builder.builderError = err +} diff --git a/statediff/testhelpers/mocks/service.go b/statediff/testhelpers/mocks/service.go new file mode 100644 index 000000000..64e48a325 --- /dev/null +++ b/statediff/testhelpers/mocks/service.go @@ -0,0 +1,322 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package mocks + +import ( + "bytes" + "errors" + "fmt" + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/rlp" + + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/statediff" + sdtypes "github.com/ethereum/go-ethereum/statediff/types" +) + +// MockStateDiffService is a mock state diff service +type MockStateDiffService struct { + sync.Mutex + Builder statediff.Builder + BlockChain *BlockChain + ReturnProtocol []p2p.Protocol + ReturnAPIs []rpc.API + BlockChan chan *types.Block + ParentBlockChan chan *types.Block + QuitChan chan bool + Subscriptions map[common.Hash]map[rpc.ID]statediff.Subscription + SubscriptionTypes map[common.Hash]statediff.Params +} + +// Protocols mock method +func (sds *MockStateDiffService) Protocols() []p2p.Protocol { + return []p2p.Protocol{} +} + +// APIs mock method +func (sds *MockStateDiffService) APIs() []rpc.API { + return []rpc.API{ + { + Namespace: statediff.APIName, + Version: statediff.APIVersion, + Service: statediff.NewPublicStateDiffAPI(sds), + Public: true, + }, + } +} + +// Loop mock method +func (sds *MockStateDiffService) Loop(chan core.ChainEvent) { + //loop through chain events until no more + for { + select { + case block := <-sds.BlockChan: + currentBlock := block + parentBlock := <-sds.ParentBlockChan + parentHash := parentBlock.Hash() + if parentBlock == nil { + log.Error("Parent block is nil, skipping this block", + "parent block hash", parentHash.String(), + "current block number", currentBlock.Number()) + continue + } + sds.streamStateDiff(currentBlock, parentBlock.Root()) + case <-sds.QuitChan: + log.Debug("Quitting the statediff block channel") + sds.close() + return + } + } +} + +// streamStateDiff method builds the state diff payload for each subscription according to their subscription type and sends them the result +func (sds *MockStateDiffService) streamStateDiff(currentBlock *types.Block, parentRoot common.Hash) { + sds.Lock() + for ty, subs := range sds.Subscriptions { + params, ok := sds.SubscriptionTypes[ty] + if !ok { + log.Error(fmt.Sprintf("subscriptions type %s do not have a parameter set associated with them", ty.Hex())) + sds.closeType(ty) + continue + } + // create payload for this subscription type + payload, err := sds.processStateDiff(currentBlock, parentRoot, params) + if err != nil { + log.Error(fmt.Sprintf("statediff processing error for subscriptions with parameters: %+v", params)) + sds.closeType(ty) + continue + } + for id, sub := range subs { + select { + case sub.PayloadChan <- *payload: + log.Debug(fmt.Sprintf("sending statediff payload to subscription %s", id)) + default: + log.Info(fmt.Sprintf("unable to send statediff payload to subscription %s; channel has no receiver", id)) + } + } + } + sds.Unlock() +} + +// StateDiffAt mock method +func (sds *MockStateDiffService) StateDiffAt(blockNumber uint64, params statediff.Params) (*statediff.Payload, error) { + currentBlock := sds.BlockChain.GetBlockByNumber(blockNumber) + log.Info(fmt.Sprintf("sending state diff at %d", blockNumber)) + if blockNumber == 0 { + return sds.processStateDiff(currentBlock, common.Hash{}, params) + } + parentBlock := sds.BlockChain.GetBlockByHash(currentBlock.ParentHash()) + return sds.processStateDiff(currentBlock, parentBlock.Root(), params) +} + +// processStateDiff method builds the state diff payload from the current block, parent state root, and provided params +func (sds *MockStateDiffService) processStateDiff(currentBlock *types.Block, parentRoot common.Hash, params statediff.Params) (*statediff.Payload, error) { + stateDiff, err := sds.Builder.BuildStateDiffObject(statediff.Args{ + NewStateRoot: currentBlock.Root(), + OldStateRoot: parentRoot, + BlockHash: currentBlock.Hash(), + BlockNumber: currentBlock.Number(), + }, params) + if err != nil { + return nil, err + } + stateDiffRlp, err := rlp.EncodeToBytes(stateDiff) + if err != nil { + return nil, err + } + return sds.newPayload(stateDiffRlp, currentBlock, params) +} + +func (sds *MockStateDiffService) newPayload(stateObject []byte, block *types.Block, params statediff.Params) (*statediff.Payload, error) { + payload := &statediff.Payload{ + StateObjectRlp: stateObject, + } + if params.IncludeBlock { + blockBuff := new(bytes.Buffer) + if err := block.EncodeRLP(blockBuff); err != nil { + return nil, err + } + payload.BlockRlp = blockBuff.Bytes() + } + if params.IncludeTD { + payload.TotalDifficulty = sds.BlockChain.GetTdByHash(block.Hash()) + } + if params.IncludeReceipts { + receiptBuff := new(bytes.Buffer) + receipts := sds.BlockChain.GetReceiptsByHash(block.Hash()) + if err := rlp.Encode(receiptBuff, receipts); err != nil { + return nil, err + } + payload.ReceiptsRlp = receiptBuff.Bytes() + } + return payload, nil +} + +// WriteStateDiffAt mock method +func (sds *MockStateDiffService) WriteStateDiffAt(blockNumber uint64, params statediff.Params) error { + // TODO: something useful here + return nil +} + +// Loop mock method +func (sds *MockStateDiffService) WriteLoop(chan core.ChainEvent) { + //loop through chain events until no more + for { + select { + case block := <-sds.BlockChan: + currentBlock := block + parentBlock := <-sds.ParentBlockChan + parentHash := parentBlock.Hash() + if parentBlock == nil { + log.Error("Parent block is nil, skipping this block", + "parent block hash", parentHash.String(), + "current block number", currentBlock.Number()) + continue + } + // TODO: + // sds.writeStateDiff(currentBlock, parentBlock.Root(), statediff.Params{}) + case <-sds.QuitChan: + log.Debug("Quitting the statediff block channel") + sds.close() + return + } + } +} + +// StateTrieAt mock method +func (sds *MockStateDiffService) StateTrieAt(blockNumber uint64, params statediff.Params) (*statediff.Payload, error) { + currentBlock := sds.BlockChain.GetBlockByNumber(blockNumber) + log.Info(fmt.Sprintf("sending state trie at %d", blockNumber)) + return sds.stateTrieAt(currentBlock, params) +} + +func (sds *MockStateDiffService) stateTrieAt(block *types.Block, params statediff.Params) (*statediff.Payload, error) { + stateNodes, err := sds.Builder.BuildStateTrieObject(block) + if err != nil { + return nil, err + } + stateTrieRlp, err := rlp.EncodeToBytes(stateNodes) + if err != nil { + return nil, err + } + return sds.newPayload(stateTrieRlp, block, params) +} + +// Subscribe is used by the API to subscribe to the service loop +func (sds *MockStateDiffService) Subscribe(id rpc.ID, sub chan<- statediff.Payload, quitChan chan<- bool, params statediff.Params) { + // Subscription type is defined as the hash of the rlp-serialized subscription params + by, err := rlp.EncodeToBytes(params) + if err != nil { + return + } + subscriptionType := crypto.Keccak256Hash(by) + // Add subscriber + sds.Lock() + if sds.Subscriptions[subscriptionType] == nil { + sds.Subscriptions[subscriptionType] = make(map[rpc.ID]statediff.Subscription) + } + sds.Subscriptions[subscriptionType][id] = statediff.Subscription{ + PayloadChan: sub, + QuitChan: quitChan, + } + sds.SubscriptionTypes[subscriptionType] = params + sds.Unlock() +} + +// Unsubscribe is used to unsubscribe from the service loop +func (sds *MockStateDiffService) Unsubscribe(id rpc.ID) error { + sds.Lock() + for ty := range sds.Subscriptions { + delete(sds.Subscriptions[ty], id) + if len(sds.Subscriptions[ty]) == 0 { + // If we removed the last subscription of this type, remove the subscription type outright + delete(sds.Subscriptions, ty) + delete(sds.SubscriptionTypes, ty) + } + } + sds.Unlock() + return nil +} + +// close is used to close all listening subscriptions +func (sds *MockStateDiffService) close() { + sds.Lock() + for ty, subs := range sds.Subscriptions { + for id, sub := range subs { + select { + case sub.QuitChan <- true: + log.Info(fmt.Sprintf("closing subscription %s", id)) + default: + log.Info(fmt.Sprintf("unable to close subscription %s; channel has no receiver", id)) + } + delete(sds.Subscriptions[ty], id) + } + delete(sds.Subscriptions, ty) + delete(sds.SubscriptionTypes, ty) + } + sds.Unlock() +} + +// Start mock method +func (sds *MockStateDiffService) Start() error { + log.Info("Starting mock statediff service") + if sds.ParentBlockChan == nil || sds.BlockChan == nil { + return errors.New("MockStateDiffingService needs to be configured with a MockParentBlockChan and MockBlockChan") + } + chainEventCh := make(chan core.ChainEvent, 10) + go sds.Loop(chainEventCh) + + return nil +} + +// Stop mock method +func (sds *MockStateDiffService) Stop() error { + log.Info("Stopping mock statediff service") + close(sds.QuitChan) + return nil +} + +// closeType is used to close all subscriptions of given type +// closeType needs to be called with subscription access locked +func (sds *MockStateDiffService) closeType(subType common.Hash) { + subs := sds.Subscriptions[subType] + for id, sub := range subs { + sendNonBlockingQuit(id, sub) + } + delete(sds.Subscriptions, subType) + delete(sds.SubscriptionTypes, subType) +} + +func (sds *MockStateDiffService) StreamCodeAndCodeHash(blockNumber uint64, outChan chan<- sdtypes.CodeAndCodeHash, quitChan chan<- bool) { + panic("implement me") +} + +func sendNonBlockingQuit(id rpc.ID, sub statediff.Subscription) { + select { + case sub.QuitChan <- true: + log.Info(fmt.Sprintf("closing subscription %s", id)) + default: + log.Info("unable to close subscription %s; channel has no receiver", id) + } +} diff --git a/statediff/testhelpers/mocks/service_test.go b/statediff/testhelpers/mocks/service_test.go new file mode 100644 index 000000000..4b8d89e41 --- /dev/null +++ b/statediff/testhelpers/mocks/service_test.go @@ -0,0 +1,238 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package mocks + +import ( + "bytes" + "math/big" + "sort" + "sync" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/statediff" + "github.com/ethereum/go-ethereum/statediff/testhelpers" + sdtypes "github.com/ethereum/go-ethereum/statediff/types" +) + +var ( + emptyStorage = make([]sdtypes.StorageNode, 0) + block0, block1 *types.Block + minerLeafKey = testhelpers.AddressToLeafKey(common.HexToAddress("0x0")) + account1, _ = rlp.EncodeToBytes(state.Account{ + Nonce: uint64(0), + Balance: big.NewInt(10000), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"), + }) + account1LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("3926db69aaced518e9b9f0f434a473e7174109c943548bb8f23be41ca76d9ad2"), + account1, + }) + minerAccount, _ = rlp.EncodeToBytes(state.Account{ + Nonce: uint64(0), + Balance: big.NewInt(2000000000000000000), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"), + }) + minerAccountLeafNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("3380c7b7ae81a58eb98d9c78de4a1fd7fd9535fc953ed2be602daaa41767312a"), + minerAccount, + }) + bankAccount, _ = rlp.EncodeToBytes(state.Account{ + Nonce: uint64(1), + Balance: big.NewInt(testhelpers.TestBankFunds.Int64() - 10000), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"), + }) + bankAccountLeafNode, _ = rlp.EncodeToBytes([]interface{}{ + common.Hex2Bytes("30bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a"), + bankAccount, + }) + mockTotalDifficulty = big.NewInt(1337) + params = statediff.Params{ + IntermediateStateNodes: false, + IncludeTD: true, + IncludeBlock: true, + IncludeReceipts: true, + } +) + +func TestAPI(t *testing.T) { + testSubscriptionAPI(t) + testHTTPAPI(t) +} + +func testSubscriptionAPI(t *testing.T) { + blocks, chain := testhelpers.MakeChain(1, testhelpers.Genesis, testhelpers.TestChainGen) + defer chain.Stop() + block0 = testhelpers.Genesis + block1 = blocks[0] + expectedBlockRlp, _ := rlp.EncodeToBytes(block1) + mockReceipt := &types.Receipt{ + BlockNumber: block1.Number(), + BlockHash: block1.Hash(), + } + expectedReceiptBytes, _ := rlp.EncodeToBytes(types.Receipts{mockReceipt}) + expectedStateDiff := statediff.StateObject{ + BlockNumber: block1.Number(), + BlockHash: block1.Hash(), + Nodes: []sdtypes.StateNode{ + { + Path: []byte{'\x05'}, + NodeType: sdtypes.Leaf, + LeafKey: minerLeafKey, + NodeValue: minerAccountLeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x0e'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.Account1LeafKey, + NodeValue: account1LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x00'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.BankLeafKey, + NodeValue: bankAccountLeafNode, + StorageNodes: emptyStorage, + }, + }, + } + expectedStateDiffBytes, _ := rlp.EncodeToBytes(expectedStateDiff) + blockChan := make(chan *types.Block) + parentBlockChain := make(chan *types.Block) + serviceQuitChan := make(chan bool) + mockBlockChain := &BlockChain{} + mockBlockChain.SetReceiptsForHash(block1.Hash(), types.Receipts{mockReceipt}) + mockBlockChain.SetTdByHash(block1.Hash(), mockTotalDifficulty) + mockService := MockStateDiffService{ + Mutex: sync.Mutex{}, + Builder: statediff.NewBuilder(chain.StateCache()), + BlockChan: blockChan, + BlockChain: mockBlockChain, + ParentBlockChan: parentBlockChain, + QuitChan: serviceQuitChan, + Subscriptions: make(map[common.Hash]map[rpc.ID]statediff.Subscription), + SubscriptionTypes: make(map[common.Hash]statediff.Params), + } + mockService.Start() + id := rpc.NewID() + payloadChan := make(chan statediff.Payload) + quitChan := make(chan bool) + mockService.Subscribe(id, payloadChan, quitChan, params) + blockChan <- block1 + parentBlockChain <- block0 + + sort.Slice(expectedStateDiffBytes, func(i, j int) bool { return expectedStateDiffBytes[i] < expectedStateDiffBytes[j] }) + select { + case payload := <-payloadChan: + if !bytes.Equal(payload.BlockRlp, expectedBlockRlp) { + t.Errorf("payload does not have expected block\r\nactual block rlp: %v\r\nexpected block rlp: %v", payload.BlockRlp, expectedBlockRlp) + } + sort.Slice(payload.StateObjectRlp, func(i, j int) bool { return payload.StateObjectRlp[i] < payload.StateObjectRlp[j] }) + if !bytes.Equal(payload.StateObjectRlp, expectedStateDiffBytes) { + t.Errorf("payload does not have expected state diff\r\nactual state diff rlp: %v\r\nexpected state diff rlp: %v", payload.StateObjectRlp, expectedStateDiffBytes) + } + if !bytes.Equal(expectedReceiptBytes, payload.ReceiptsRlp) { + t.Errorf("payload does not have expected receipts\r\nactual receipt rlp: %v\r\nexpected receipt rlp: %v", payload.ReceiptsRlp, expectedReceiptBytes) + } + if !bytes.Equal(payload.TotalDifficulty.Bytes(), mockTotalDifficulty.Bytes()) { + t.Errorf("payload does not have expected total difficulty\r\nactual td: %d\r\nexpected td: %d", payload.TotalDifficulty.Int64(), mockTotalDifficulty.Int64()) + } + case <-quitChan: + t.Errorf("channel quit before delivering payload") + } +} + +func testHTTPAPI(t *testing.T) { + blocks, chain := testhelpers.MakeChain(1, testhelpers.Genesis, testhelpers.TestChainGen) + defer chain.Stop() + block0 = testhelpers.Genesis + block1 = blocks[0] + expectedBlockRlp, _ := rlp.EncodeToBytes(block1) + mockReceipt := &types.Receipt{ + BlockNumber: block1.Number(), + BlockHash: block1.Hash(), + } + expectedReceiptBytes, _ := rlp.EncodeToBytes(types.Receipts{mockReceipt}) + expectedStateDiff := statediff.StateObject{ + BlockNumber: block1.Number(), + BlockHash: block1.Hash(), + Nodes: []sdtypes.StateNode{ + { + Path: []byte{'\x05'}, + NodeType: sdtypes.Leaf, + LeafKey: minerLeafKey, + NodeValue: minerAccountLeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x0e'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.Account1LeafKey, + NodeValue: account1LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x00'}, + NodeType: sdtypes.Leaf, + LeafKey: testhelpers.BankLeafKey, + NodeValue: bankAccountLeafNode, + StorageNodes: emptyStorage, + }, + }, + } + expectedStateDiffBytes, _ := rlp.EncodeToBytes(expectedStateDiff) + mockBlockChain := &BlockChain{} + mockBlockChain.SetBlocksForHashes(map[common.Hash]*types.Block{ + block0.Hash(): block0, + block1.Hash(): block1, + }) + mockBlockChain.SetBlockForNumber(block1, block1.Number().Uint64()) + mockBlockChain.SetReceiptsForHash(block1.Hash(), types.Receipts{mockReceipt}) + mockBlockChain.SetTdByHash(block1.Hash(), big.NewInt(1337)) + mockService := MockStateDiffService{ + Mutex: sync.Mutex{}, + Builder: statediff.NewBuilder(chain.StateCache()), + BlockChain: mockBlockChain, + } + payload, err := mockService.StateDiffAt(block1.Number().Uint64(), params) + if err != nil { + t.Error(err) + } + sort.Slice(payload.StateObjectRlp, func(i, j int) bool { return payload.StateObjectRlp[i] < payload.StateObjectRlp[j] }) + sort.Slice(expectedStateDiffBytes, func(i, j int) bool { return expectedStateDiffBytes[i] < expectedStateDiffBytes[j] }) + if !bytes.Equal(payload.BlockRlp, expectedBlockRlp) { + t.Errorf("payload does not have expected block\r\nactual block rlp: %v\r\nexpected block rlp: %v", payload.BlockRlp, expectedBlockRlp) + } + if !bytes.Equal(payload.StateObjectRlp, expectedStateDiffBytes) { + t.Errorf("payload does not have expected state diff\r\nactual state diff rlp: %v\r\nexpected state diff rlp: %v", payload.StateObjectRlp, expectedStateDiffBytes) + } + if !bytes.Equal(expectedReceiptBytes, payload.ReceiptsRlp) { + t.Errorf("payload does not have expected receipts\r\nactual receipt rlp: %v\r\nexpected receipt rlp: %v", payload.ReceiptsRlp, expectedReceiptBytes) + } + if !bytes.Equal(payload.TotalDifficulty.Bytes(), mockTotalDifficulty.Bytes()) { + t.Errorf("paylaod does not have the expected total difficulty\r\nactual td: %d\r\nexpected td: %d", payload.TotalDifficulty.Int64(), mockTotalDifficulty.Int64()) + } +} diff --git a/statediff/testhelpers/test_data.go b/statediff/testhelpers/test_data.go new file mode 100644 index 000000000..cc5374da9 --- /dev/null +++ b/statediff/testhelpers/test_data.go @@ -0,0 +1,73 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package testhelpers + +import ( + "math/big" + "math/rand" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/rlp" +) + +// AddressToLeafKey hashes an returns an address +func AddressToLeafKey(address common.Address) []byte { + return crypto.Keccak256(address[:]) +} + +// AddressToEncodedPath hashes an address and appends the even-number leaf flag to it +func AddressToEncodedPath(address common.Address) []byte { + addrHash := crypto.Keccak256(address[:]) + decodedPath := append(EvenLeafFlag, addrHash...) + return decodedPath +} + +// Test variables +var ( + EvenLeafFlag = []byte{byte(2) << 4} + BlockNumber = big.NewInt(rand.Int63()) + BlockHash = "0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73" + NullCodeHash = crypto.Keccak256Hash([]byte{}) + StoragePath = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes() + StorageKey = common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001").Bytes() + StorageValue = common.Hex2Bytes("0x03") + NullHash = common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000") + + Testdb = rawdb.NewMemoryDatabase() + TestBankKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + TestBankAddress = crypto.PubkeyToAddress(TestBankKey.PublicKey) //0x71562b71999873DB5b286dF957af199Ec94617F7 + BankLeafKey = AddressToLeafKey(TestBankAddress) + TestBankFunds = big.NewInt(100000000) + Genesis = core.GenesisBlockForTesting(Testdb, TestBankAddress, TestBankFunds) + + Account1Key, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") + Account2Key, _ = crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee") + Account1Addr = crypto.PubkeyToAddress(Account1Key.PublicKey) //0x703c4b2bD70c169f5717101CaeE543299Fc946C7 + Account2Addr = crypto.PubkeyToAddress(Account2Key.PublicKey) //0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e + Account1LeafKey = AddressToLeafKey(Account1Addr) + Account2LeafKey = AddressToLeafKey(Account2Addr) + ContractCode = common.Hex2Bytes("608060405234801561001057600080fd5b50336000806101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff1602179055506040518060200160405280600160ff16815250600190600161007492919061007a565b506100e4565b82606481019282156100ae579160200282015b828111156100ad578251829060ff1690559160200191906001019061008d565b5b5090506100bb91906100bf565b5090565b6100e191905b808211156100dd5760008160009055506001016100c5565b5090565b90565b6101ca806100f36000396000f3fe608060405234801561001057600080fd5b50600436106100365760003560e01c806343d726d61461003b578063c16431b914610045575b600080fd5b61004361007d565b005b61007b6004803603604081101561005b57600080fd5b81019080803590602001909291908035906020019092919050505061015c565b005b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614610122576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260228152602001806101746022913960400191505060405180910390fd5b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16ff5b806001836064811061016a57fe5b0181905550505056fe4f6e6c79206f776e65722063616e2063616c6c20746869732066756e6374696f6e2ea265627a7a72305820e3747183708fb6bff3f6f7a80fb57dcc1c19f83f9cb25457a3ed5c0424bde66864736f6c634300050a0032") + ByteCodeAfterDeployment = common.Hex2Bytes("608060405234801561001057600080fd5b50600436106100365760003560e01c806343d726d61461003b578063c16431b914610045575b600080fd5b61004361007d565b005b61007b6004803603604081101561005b57600080fd5b81019080803590602001909291908035906020019092919050505061015c565b005b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614610122576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260228152602001806101746022913960400191505060405180910390fd5b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16ff5b806001836064811061016a57fe5b0181905550505056fe4f6e6c79206f776e65722063616e2063616c6c20746869732066756e6374696f6e2ea265627a7a72305820e3747183708fb6bff3f6f7a80fb57dcc1c19f83f9cb25457a3ed5c0424bde66864736f6c634300050a0032") + CodeHash = common.HexToHash("0xaaea5efba4fd7b45d7ec03918ac5d8b31aa93b48986af0e6b591f0f087c80127") + ContractAddr common.Address + + EmptyRootNode, _ = rlp.EncodeToBytes([]byte{}) + EmptyContractRoot = crypto.Keccak256Hash(EmptyRootNode) +) diff --git a/statediff/types.go b/statediff/types.go new file mode 100644 index 000000000..148567dd7 --- /dev/null +++ b/statediff/types.go @@ -0,0 +1,113 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package statediff + +import ( + "encoding/json" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/statediff/types" +) + +// Subscription struct holds our subscription channels +type Subscription struct { + PayloadChan chan<- Payload + QuitChan chan<- bool +} + +// DBParams holds params for Postgres db connection +type DBParams struct { + ConnectionURL string + ID string + ClientName string +} + +// Params is used to carry in parameters from subscribing/requesting clients configuration +type Params struct { + IntermediateStateNodes bool + IntermediateStorageNodes bool + IncludeBlock bool + IncludeReceipts bool + IncludeTD bool + IncludeCode bool + WatchedAddresses []common.Address + WatchedStorageSlots []common.Hash +} + +// Args bundles the arguments for the state diff builder +type Args struct { + OldStateRoot, NewStateRoot, BlockHash common.Hash + BlockNumber *big.Int +} + +type StateRoots struct { + OldStateRoot, NewStateRoot common.Hash +} + +// Payload packages the data to send to statediff subscriptions +type Payload struct { + BlockRlp []byte `json:"blockRlp"` + TotalDifficulty *big.Int `json:"totalDifficulty"` + ReceiptsRlp []byte `json:"receiptsRlp"` + StateObjectRlp []byte `json:"stateObjectRlp" gencodec:"required"` + + encoded []byte + err error +} + +func (sd *Payload) ensureEncoded() { + if sd.encoded == nil && sd.err == nil { + sd.encoded, sd.err = json.Marshal(sd) + } +} + +// Length to implement Encoder interface for Payload +func (sd *Payload) Length() int { + sd.ensureEncoded() + return len(sd.encoded) +} + +// Encode to implement Encoder interface for Payload +func (sd *Payload) Encode() ([]byte, error) { + sd.ensureEncoded() + return sd.encoded, sd.err +} + +// StateObject is the final output structure from the builder +type StateObject struct { + BlockNumber *big.Int `json:"blockNumber" gencodec:"required"` + BlockHash common.Hash `json:"blockHash" gencodec:"required"` + Nodes []types.StateNode `json:"nodes" gencodec:"required"` + CodeAndCodeHashes []types.CodeAndCodeHash `json:"codeMapping"` +} + +// AccountMap is a mapping of hex encoded path => account wrapper +type AccountMap map[string]accountWrapper + +// accountWrapper is used to temporary associate the unpacked node with its raw values +type accountWrapper struct { + Account *state.Account + NodeType types.NodeType + Path []byte + NodeValue []byte + LeafKey []byte +} diff --git a/statediff/types/types.go b/statediff/types/types.go new file mode 100644 index 000000000..08e2124fa --- /dev/null +++ b/statediff/types/types.go @@ -0,0 +1,61 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package types + +import "github.com/ethereum/go-ethereum/common" + +// NodeType for explicitly setting type of node +type NodeType string + +const ( + Unknown NodeType = "Unknown" + Leaf NodeType = "Leaf" + Extension NodeType = "Extension" + Branch NodeType = "Branch" + Removed NodeType = "Removed" // used to represent pathes which have been emptied +) + +// StateNode holds the data for a single state diff node +type StateNode struct { + NodeType NodeType `json:"nodeType" gencodec:"required"` + Path []byte `json:"path" gencodec:"required"` + NodeValue []byte `json:"value" gencodec:"required"` + StorageNodes []StorageNode `json:"storage"` + LeafKey []byte `json:"leafKey"` +} + +// StorageNode holds the data for a single storage diff node +type StorageNode struct { + NodeType NodeType `json:"nodeType" gencodec:"required"` + Path []byte `json:"path" gencodec:"required"` + NodeValue []byte `json:"value" gencodec:"required"` + LeafKey []byte `json:"leafKey"` +} + +// CodeAndCodeHash struct for holding codehash => code mappings +// we can't use an actual map because they are not rlp serializable +type CodeAndCodeHash struct { + Hash common.Hash `json:"codeHash"` + Code []byte `json:"code"` +} + +type StateNodeSink func(StateNode) error +type StorageNodeSink func(StorageNode) error +type CodeSink func(CodeAndCodeHash) error diff --git a/trie/encoding.go b/trie/encoding.go index 8ee0022ef..3a5f632e2 100644 --- a/trie/encoding.go +++ b/trie/encoding.go @@ -34,6 +34,11 @@ package trie // in the case of an odd number. All remaining nibbles (now an even number) fit properly // into the remaining bytes. Compact encoding is used for nodes stored on disk. +// HexToCompact converts a hex path to the compact encoded format +func HexToCompact(hex []byte) []byte { + return hexToCompact(hex) +} + func hexToCompact(hex []byte) []byte { terminator := byte(0) if hasTerm(hex) { @@ -55,8 +60,8 @@ func hexToCompact(hex []byte) []byte { // needed for the representation func hexToCompactInPlace(hex []byte) int { var ( - hexLen = len(hex) // length of the hex input - firstByte = byte(0) + hexLen= len(hex) // length of the hex input + firstByte= byte(0) ) // Check if we have a terminator there if hexLen > 0 && hex[hexLen-1] == 16 { @@ -64,9 +69,9 @@ func hexToCompactInPlace(hex []byte) int { hexLen-- // last part was the terminator, ignore that } var ( - binLen = hexLen/2 + 1 - ni = 0 // index in hex - bi = 1 // index in bin (compact) + binLen= hexLen/2 + 1 + ni= 0 // index in hex + bi= 1 // index in bin (compact) ) if hexLen&1 == 1 { firstByte |= 1 << 4 // odd flag @@ -80,6 +85,11 @@ func hexToCompactInPlace(hex []byte) int { return binLen } +// CompactToHex converts a compact encoded path to hex format +func CompactToHex(compact []byte) []byte { + return compactToHex(compact) +} + func compactToHex(compact []byte) []byte { if len(compact) == 0 { return compact @@ -105,9 +115,9 @@ func keybytesToHex(str []byte) []byte { return nibbles } -// hexToKeybytes turns hex nibbles into key bytes. +// hexToKeyBytes turns hex nibbles into key bytes. // This can only be used for keys of even length. -func hexToKeybytes(hex []byte) []byte { +func hexToKeyBytes(hex []byte) []byte { if hasTerm(hex) { hex = hex[:len(hex)-1] } diff --git a/trie/encoding_test.go b/trie/encoding_test.go index 16393313f..7ade0a095 100644 --- a/trie/encoding_test.go +++ b/trie/encoding_test.go @@ -71,8 +71,8 @@ func TestHexKeybytes(t *testing.T) { if h := keybytesToHex(test.key); !bytes.Equal(h, test.hexOut) { t.Errorf("keybytesToHex(%x) -> %x, want %x", test.key, h, test.hexOut) } - if k := hexToKeybytes(test.hexIn); !bytes.Equal(k, test.key) { - t.Errorf("hexToKeybytes(%x) -> %x, want %x", test.hexIn, k, test.key) + if k := hexToKeyBytes(test.hexIn); !bytes.Equal(k, test.key) { + t.Errorf("hexToKeyBytes(%x) -> %x, want %x", test.hexIn, k, test.key) } } } @@ -135,6 +135,6 @@ func BenchmarkKeybytesToHex(b *testing.B) { func BenchmarkHexToKeybytes(b *testing.B) { testBytes := []byte{7, 6, 6, 5, 7, 2, 6, 2, 16} for i := 0; i < b.N; i++ { - hexToKeybytes(testBytes) + hexToKeyBytes(testBytes) } } diff --git a/trie/iterator.go b/trie/iterator.go index 76d437c40..c4fce17df 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -164,7 +164,7 @@ func (it *nodeIterator) Leaf() bool { func (it *nodeIterator) LeafKey() []byte { if len(it.stack) > 0 { if _, ok := it.stack[len(it.stack)-1].node.(valueNode); ok { - return hexToKeybytes(it.path) + return hexToKeyBytes(it.path) } } panic("not at leaf") diff --git a/trie/sync.go b/trie/sync.go index bc93ddd3f..a39351b06 100644 --- a/trie/sync.go +++ b/trie/sync.go @@ -82,7 +82,7 @@ func newSyncPath(path []byte) SyncPath { if len(path) < 64 { return SyncPath{hexToCompact(path)} } - return SyncPath{hexToKeybytes(path[:64]), hexToCompact(path[64:])} + return SyncPath{hexToKeyBytes(path[:64]), hexToCompact(path[64:])} } // SyncResult is a response with requested data along with it's hash.