From b454b61777f006b5c948e1b2dfcd49cfeac743dd Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Fri, 4 Oct 2019 13:54:37 -0500 Subject: [PATCH] factor our state diff fetcher for more general use (e.g. by the super node) --- .../shared/fetcher/state_diff_fetcher.go | 72 +++++++++++ .../shared/fetcher/state_diff_fetcher_test.go | 117 ++++++++++++++++++ libraries/shared/storage/backfill.go | 35 ++---- 3 files changed, 199 insertions(+), 25 deletions(-) create mode 100644 libraries/shared/fetcher/state_diff_fetcher.go create mode 100644 libraries/shared/fetcher/state_diff_fetcher_test.go diff --git a/libraries/shared/fetcher/state_diff_fetcher.go b/libraries/shared/fetcher/state_diff_fetcher.go new file mode 100644 index 00000000..087655c2 --- /dev/null +++ b/libraries/shared/fetcher/state_diff_fetcher.go @@ -0,0 +1,72 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package fetcher + +import ( + "github.com/ethereum/go-ethereum/statediff" + + "github.com/vulcanize/vulcanizedb/pkg/geth/client" +) + +// IStateDiffFetcher is the state diff fetching interface +type IStateDiffFetcher interface { + FetchStateDiffsAt(blockHeights []uint64) ([]*statediff.Payload, error) +} + +// BatchClient is an interface to a batch-fetching geth rpc client; created to allow mock insertion +type BatchClient interface { + BatchCall(batch []client.BatchElem) error +} + +// StateDiffFetcher is the state diff fetching struct +type StateDiffFetcher struct { + client BatchClient +} + +const method = "statediff_stateDiffAt" + +// NewStateDiffFetcher returns a IStateDiffFetcher +func NewStateDiffFetcher(bc BatchClient) IStateDiffFetcher { + return &StateDiffFetcher{ + client: bc, + } +} + +// FetchStateDiffsAt fetches the statediff payloads at the given block heights +// Calls StateDiffAt(ctx context.Context, blockNumber uint64) (*Payload, error) +func (sdf *StateDiffFetcher) FetchStateDiffsAt(blockHeights []uint64) ([]*statediff.Payload, error) { + batch := make([]client.BatchElem, 0) + for _, height := range blockHeights { + batch = append(batch, client.BatchElem{ + Method: method, + Args: []interface{}{height}, + Result: new(statediff.Payload), + }) + } + batchErr := sdf.client.BatchCall(batch) + if batchErr != nil { + return nil, batchErr + } + results := make([]*statediff.Payload, 0, len(blockHeights)) + for _, batchElem := range batch { + if batchElem.Error != nil { + return nil, batchElem.Error + } + results = append(results, batchElem.Result.(*statediff.Payload)) + } + return results, nil +} diff --git a/libraries/shared/fetcher/state_diff_fetcher_test.go b/libraries/shared/fetcher/state_diff_fetcher_test.go new file mode 100644 index 00000000..7ff33899 --- /dev/null +++ b/libraries/shared/fetcher/state_diff_fetcher_test.go @@ -0,0 +1,117 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package fetcher_test + +import ( + "bytes" + "encoding/json" + "errors" + + "github.com/vulcanize/vulcanizedb/libraries/shared/fetcher" + + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/statediff" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/libraries/shared/test_data" + "github.com/vulcanize/vulcanizedb/pkg/geth/client" +) + +type mockClient struct { + MappedStateDiffAt map[uint64][]byte +} + +// SetReturnDiffAt method to set what statediffs the mock client returns +func (mc *mockClient) SetReturnDiffAt(height uint64, diffPayload statediff.Payload) error { + if mc.MappedStateDiffAt == nil { + mc.MappedStateDiffAt = make(map[uint64][]byte) + } + by, err := json.Marshal(diffPayload) + if err != nil { + return err + } + mc.MappedStateDiffAt[height] = by + return nil +} + +// BatchCall mockClient method to simulate batch call to geth +func (mc *mockClient) BatchCall(batch []client.BatchElem) error { + if mc.MappedStateDiffAt == nil { + return errors.New("mockclient needs to be initialized with statediff payloads and errors") + } + for _, batchElem := range batch { + if len(batchElem.Args) != 1 { + return errors.New("expected batch elem to contain single argument") + } + blockHeight, ok := batchElem.Args[0].(uint64) + if !ok { + return errors.New("expected batch elem argument to be a uint64") + } + err := json.Unmarshal(mc.MappedStateDiffAt[blockHeight], batchElem.Result) + if err != nil { + return err + } + } + return nil +} + +var _ = Describe("StateDiffFetcher", func() { + Describe("FetchStateDiffsAt", func() { + var ( + mc *mockClient + stateDiffFetcher fetcher.IStateDiffFetcher + ) + BeforeEach(func() { + mc = new(mockClient) + setDiffAtErr1 := mc.SetReturnDiffAt(test_data.BlockNumber.Uint64(), test_data.MockStatediffPayload) + Expect(setDiffAtErr1).ToNot(HaveOccurred()) + setDiffAtErr2 := mc.SetReturnDiffAt(test_data.BlockNumber2.Uint64(), test_data.MockStatediffPayload2) + Expect(setDiffAtErr2).ToNot(HaveOccurred()) + stateDiffFetcher = fetcher.NewStateDiffFetcher(mc) + }) + It("Batch calls statediff_stateDiffAt", func() { + blockHeights := []uint64{ + test_data.BlockNumber.Uint64(), + test_data.BlockNumber2.Uint64(), + } + stateDiffPayloads, fetchErr := stateDiffFetcher.FetchStateDiffsAt(blockHeights) + Expect(fetchErr).ToNot(HaveOccurred()) + Expect(len(stateDiffPayloads)).To(Equal(2)) + // Can only rlp encode the slice of diffs as part of a struct + // Rlp encoding allows us to compare content of the slices when the order in the slice may vary + expectedPayloadsStruct := struct { + payloads []*statediff.Payload + }{ + []*statediff.Payload{ + &test_data.MockStatediffPayload, + &test_data.MockStatediffPayload2, + }, + } + expectedPayloadsBytes, rlpErr1 := rlp.EncodeToBytes(expectedPayloadsStruct) + Expect(rlpErr1).ToNot(HaveOccurred()) + receivedPayloadsStruct := struct { + payloads []*statediff.Payload + }{ + stateDiffPayloads, + } + receivedPayloadsBytes, rlpErr2 := rlp.EncodeToBytes(receivedPayloadsStruct) + Expect(rlpErr2).ToNot(HaveOccurred()) + Expect(bytes.Equal(expectedPayloadsBytes, receivedPayloadsBytes)).To(BeTrue()) + }) + }) +}) diff --git a/libraries/shared/storage/backfill.go b/libraries/shared/storage/backfill.go index a0aaadef..70a14d3b 100644 --- a/libraries/shared/storage/backfill.go +++ b/libraries/shared/storage/backfill.go @@ -27,8 +27,8 @@ import ( "github.com/ethereum/go-ethereum/statediff" "github.com/sirupsen/logrus" + "github.com/vulcanize/vulcanizedb/libraries/shared/fetcher" "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils" - "github.com/vulcanize/vulcanizedb/pkg/geth/client" ) // IBackFiller is the backfilling interface @@ -36,14 +36,9 @@ type IBackFiller interface { BackFill(bfa BackFillerArgs) (map[common.Hash][]utils.StorageDiff, error) } -// BatchClient is an interface to a batch-fetching geth rpc client; created to allow mock insertion -type BatchClient interface { - BatchCall(batch []client.BatchElem) error -} - // BackFiller is the backfilling struct type BackFiller struct { - client BatchClient + sdf fetcher.IStateDiffFetcher } // BackFillerArgs are used to pass configuration params to the backfiller @@ -54,12 +49,10 @@ type BackFillerArgs struct { EndingBlock uint64 } -const method = "statediff_stateDiffAt" - // NewStorageBackFiller returns a IBackFiller -func NewStorageBackFiller(bc BatchClient) IBackFiller { +func NewStorageBackFiller(bc fetcher.BatchClient) IBackFiller { return &BackFiller{ - client: bc, + sdf: fetcher.NewStateDiffFetcher(bc), } } @@ -70,23 +63,15 @@ func (bf *BackFiller) BackFill(bfa BackFillerArgs) (map[common.Hash][]utils.Stor if bfa.EndingBlock < bfa.StartingBlock { return nil, errors.New("backfill: ending block number needs to be greater than starting block number") } - batch := make([]client.BatchElem, 0) + blockHeights := make([]uint64, 0, bfa.EndingBlock-bfa.StartingBlock+1) for i := bfa.StartingBlock; i <= bfa.EndingBlock; i++ { - batch = append(batch, client.BatchElem{ - Method: method, - Args: []interface{}{i}, - Result: new(statediff.Payload), - }) + blockHeights = append(blockHeights, i) } - batchErr := bf.client.BatchCall(batch) - if batchErr != nil { - return nil, batchErr + payloads, err := bf.sdf.FetchStateDiffsAt(blockHeights) + if err != nil { + return nil, err } - for _, batchElem := range batch { - payload := batchElem.Result.(*statediff.Payload) - if batchElem.Error != nil { - return nil, batchElem.Error - } + for _, payload := range payloads { block := new(types.Block) blockDecodeErr := rlp.DecodeBytes(payload.BlockRlp, block) if blockDecodeErr != nil {