From e3c2e922659aa740eb499f098b10975bcd7ba582 Mon Sep 17 00:00:00 2001 From: erikdies Date: Wed, 29 Jun 2022 11:30:30 -0400 Subject: [PATCH 1/7] Create issues-notion-sync.yml --- .github/workflows/issues-notion-sync.yml | 29 ++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .github/workflows/issues-notion-sync.yml diff --git a/.github/workflows/issues-notion-sync.yml b/.github/workflows/issues-notion-sync.yml new file mode 100644 index 00000000..551362b0 --- /dev/null +++ b/.github/workflows/issues-notion-sync.yml @@ -0,0 +1,29 @@ +name: Notion Sync + +on: + workflow_dispatch: + issues: + types: + [ + opened, + edited, + labeled, + unlabeled, + assigned, + unassigned, + milestoned, + demilestoned, + reopened, + closed, + ] + +jobs: + notion_job: + runs-on: ubuntu-latest + name: Add GitHub Issues to Notion + steps: + - name: Add GitHub Issues to Notion + uses: vulcanize/notion-github-action@v1.2.4-issueid + with: + notion-token: ${{ secrets.NOTION_TOKEN }} + notion-db: ${{ secrets.NOTION_DATABASE }} From d64ac5792c4f30ee90f4688a2b167a8e706a0089 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Tue, 10 Jan 2023 12:04:28 -0600 Subject: [PATCH 2/7] Add forwardGetStorageAt setting. --- cmd/serve.go | 1 + pkg/eth/api.go | 40 +++++++++++++++++++++++++++------------ pkg/eth/api_test.go | 2 +- pkg/eth/eth_state_test.go | 2 +- pkg/eth/ipld_retriever.go | 3 +++ pkg/serve/config.go | 34 ++++++++++++++++++--------------- pkg/serve/service.go | 5 ++++- 7 files changed, 57 insertions(+), 30 deletions(-) diff --git a/cmd/serve.go b/cmd/serve.go index d3fbe243..f4973947 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -395,6 +395,7 @@ func init() { viper.BindPFlag("ethereum.chainConfig", serveCmd.PersistentFlags().Lookup("eth-chain-config")) viper.BindPFlag("ethereum.supportsStateDiff", serveCmd.PersistentFlags().Lookup("eth-supports-state-diff")) viper.BindPFlag("ethereum.forwardEthCalls", serveCmd.PersistentFlags().Lookup("eth-forward-eth-calls")) + viper.BindPFlag("ethereum.forwardGetStorageAt", serveCmd.PersistentFlags().Lookup("eth-forward-get-storage-at")) viper.BindPFlag("ethereum.proxyOnError", serveCmd.PersistentFlags().Lookup("eth-proxy-on-error")) // groupcache flags diff --git a/pkg/eth/api.go b/pkg/eth/api.go index 34451e95..9757be6e 100644 --- a/pkg/eth/api.go +++ b/pkg/eth/api.go @@ -57,18 +57,22 @@ type PublicEthAPI struct { B *Backend // Proxy node for forwarding cache misses - supportsStateDiff bool // Whether the remote node supports the statediff_writeStateDiffAt endpoint, if it does we can fill the local cache when we hit a miss - rpc *rpc.Client - ethClient *ethclient.Client - forwardEthCalls bool // if true, forward eth_call calls directly to the configured proxy node - proxyOnError bool // turn on regular proxy fall-through on errors; needed to test difference between direct and indirect fall-through + supportsStateDiff bool // Whether the remote node supports the statediff_writeStateDiffAt endpoint, if it does we can fill the local cache when we hit a miss + rpc *rpc.Client + ethClient *ethclient.Client + forwardEthCalls bool // if true, forward eth_call calls directly to the configured proxy node + forwardGetStorageAt bool // if true, forward eth_getStorageAt calls directly to the configured proxy node + proxyOnError bool // turn on regular proxy fall-through on errors; needed to test difference between direct and indirect fall-through } // NewPublicEthAPI creates a new PublicEthAPI with the provided underlying Backend -func NewPublicEthAPI(b *Backend, client *rpc.Client, supportsStateDiff, forwardEthCalls, proxyOnError bool) (*PublicEthAPI, error) { +func NewPublicEthAPI(b *Backend, client *rpc.Client, supportsStateDiff, forwardEthCalls, forwardGetStorageAt, proxyOnError bool) (*PublicEthAPI, error) { if forwardEthCalls && client == nil { return nil, errors.New("ipld-eth-server is configured to forward eth_calls to proxy node but no proxy node is configured") } + if forwardGetStorageAt && client == nil { + return nil, errors.New("ipld-eth-server is configured to forward eth_getStorageAt to proxy node but no proxy node is configured") + } if proxyOnError && client == nil { return nil, errors.New("ipld-eth-server is configured to forward all calls to proxy node on errors but no proxy node is configured") } @@ -77,12 +81,13 @@ func NewPublicEthAPI(b *Backend, client *rpc.Client, supportsStateDiff, forwardE ethClient = ethclient.NewClient(client) } return &PublicEthAPI{ - B: b, - supportsStateDiff: supportsStateDiff, - rpc: client, - ethClient: ethClient, - forwardEthCalls: forwardEthCalls, - proxyOnError: proxyOnError, + B: b, + supportsStateDiff: supportsStateDiff, + rpc: client, + ethClient: ethClient, + forwardEthCalls: forwardEthCalls, + forwardGetStorageAt: forwardGetStorageAt, + proxyOnError: proxyOnError, }, nil } @@ -720,6 +725,13 @@ func (pea *PublicEthAPI) localGetBalance(ctx context.Context, address common.Add // block number. The rpc.LatestBlockNumber and rpc.PendingBlockNumber meta block // numbers are also allowed. func (pea *PublicEthAPI) GetStorageAt(ctx context.Context, address common.Address, key string, blockNrOrHash rpc.BlockNumberOrHash) (hexutil.Bytes, error) { + if pea.forwardGetStorageAt { + var res hexutil.Bytes + // If forwarding all getStorageAt calls, don't request statediffing. + if err := pea.rpc.CallContext(ctx, &res, "eth_getStorageAt", address, key, blockNrOrHash); res != nil && err == nil { + return res, nil + } + } storageVal, err := pea.B.GetStorageByNumberOrHash(ctx, address, common.HexToHash(key), blockNrOrHash) if storageVal != nil && err == nil { var value common.Hash @@ -735,8 +747,10 @@ func (pea *PublicEthAPI) GetStorageAt(ctx context.Context, address common.Addres return value[:], nil } if pea.proxyOnError { + logrus.Warnf("Missing eth_getStorageAt(%s, %s, %s)", address.Hash().String(), key, blockNrOrHash.String()) var res hexutil.Bytes if err := pea.rpc.CallContext(ctx, &res, "eth_getStorageAt", address, key, blockNrOrHash); res != nil && err == nil { + // If only proxying on error, request statediffing for the missing data. go pea.writeStateDiffAtOrFor(blockNrOrHash) return res, nil } @@ -1065,6 +1079,7 @@ func (pea *PublicEthAPI) writeStateDiffAt(height int64) { IncludeTD: true, IncludeCode: true, } + logrus.Debugf("Calling statediff_writeStateDiffAt(%d)", height) if err := pea.rpc.CallContext(ctx, &data, "statediff_writeStateDiffAt", uint64(height), params); err != nil { logrus.Errorf("writeStateDiffAt %d faild with err %s", height, err.Error()) } @@ -1087,6 +1102,7 @@ func (pea *PublicEthAPI) writeStateDiffFor(blockHash common.Hash) { IncludeTD: true, IncludeCode: true, } + logrus.Debugf("Calling statediff_writeStateDiffFor(%s)", blockHash.Hex()) if err := pea.rpc.CallContext(ctx, &data, "statediff_writeStateDiffFor", blockHash, params); err != nil { logrus.Errorf("writeStateDiffFor %s faild with err %s", blockHash.Hex(), err.Error()) } diff --git a/pkg/eth/api_test.go b/pkg/eth/api_test.go index b3ef9d9f..a5b35c7f 100644 --- a/pkg/eth/api_test.go +++ b/pkg/eth/api_test.go @@ -213,7 +213,7 @@ var _ = Describe("API", func() { }, }) Expect(err).ToNot(HaveOccurred()) - api, _ = eth.NewPublicEthAPI(backend, nil, false, false, false) + api, _ = eth.NewPublicEthAPI(backend, nil, false, false, false, false) tx, err = indexAndPublisher.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty()) Expect(err).ToNot(HaveOccurred()) diff --git a/pkg/eth/eth_state_test.go b/pkg/eth/eth_state_test.go index 999ee3c9..0d5b2e91 100644 --- a/pkg/eth/eth_state_test.go +++ b/pkg/eth/eth_state_test.go @@ -92,7 +92,7 @@ var _ = Describe("eth state reading tests", func() { }, }) Expect(err).ToNot(HaveOccurred()) - api, _ = eth.NewPublicEthAPI(backend, nil, false, false, false) + api, _ = eth.NewPublicEthAPI(backend, nil, false, false, false, false) // make the test blockchain (and state) blocks, receipts, chain = test_helpers.MakeChain(chainLength, test_helpers.Genesis, test_helpers.TestChainGen) diff --git a/pkg/eth/ipld_retriever.go b/pkg/eth/ipld_retriever.go index 3b6d4073..c1076023 100644 --- a/pkg/eth/ipld_retriever.go +++ b/pkg/eth/ipld_retriever.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" "github.com/lib/pq" + log "github.com/sirupsen/logrus" ) const ( @@ -493,6 +494,8 @@ func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageSlotAndBlockHash(add if err := r.db.Get(storageResult, RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr, stateLeafKey.Hex(), storageHash.Hex(), hash.Hex()); err != nil { return "", nil, nil, err } + log.Debugf("getStorageAt: state_leaf_key=%s, storage_hex=%s, storage_leaf_key=%s, block_hash=%s", + stateLeafKey.Hex(), key.Hex(), storageHash.Hex(), hash.Hex()) if storageResult.StateLeafRemoved || storageResult.NodeType == removedNode { return "", EmptyNodeValue, EmptyNodeValue, nil } diff --git a/pkg/serve/config.go b/pkg/serve/config.go index 0746b549..7e4adb7e 100644 --- a/pkg/serve/config.go +++ b/pkg/serve/config.go @@ -46,12 +46,13 @@ const ( SERVER_MAX_OPEN_CONNECTIONS = "SERVER_MAX_OPEN_CONNECTIONS" SERVER_MAX_CONN_LIFETIME = "SERVER_MAX_CONN_LIFETIME" - ETH_DEFAULT_SENDER_ADDR = "ETH_DEFAULT_SENDER_ADDR" - ETH_RPC_GAS_CAP = "ETH_RPC_GAS_CAP" - ETH_CHAIN_CONFIG = "ETH_CHAIN_CONFIG" - ETH_SUPPORTS_STATEDIFF = "ETH_SUPPORTS_STATEDIFF" - ETH_FORWARD_ETH_CALLS = "ETH_FORWARD_ETH_CALLS" - ETH_PROXY_ON_ERROR = "ETH_PROXY_ON_ERROR" + ETH_DEFAULT_SENDER_ADDR = "ETH_DEFAULT_SENDER_ADDR" + ETH_RPC_GAS_CAP = "ETH_RPC_GAS_CAP" + ETH_CHAIN_CONFIG = "ETH_CHAIN_CONFIG" + ETH_SUPPORTS_STATEDIFF = "ETH_SUPPORTS_STATEDIFF" + ETH_FORWARD_ETH_CALLS = "ETH_FORWARD_ETH_CALLS" + ETH_FORWARD_GET_STORAGE_AT = "ETH_FORWARD_GET_STORAGE_AT" + ETH_PROXY_ON_ERROR = "ETH_PROXY_ON_ERROR" VALIDATOR_ENABLED = "VALIDATOR_ENABLED" VALIDATOR_EVERY_NTH_BLOCK = "VALIDATOR_EVERY_NTH_BLOCK" @@ -80,15 +81,16 @@ type Config struct { TracingHttpEndpoint string TracingPostgraphileEndpoint string - ChainConfig *params.ChainConfig - DefaultSender *common.Address - RPCGasCap *big.Int - EthHttpEndpoint string - Client *rpc.Client - SupportStateDiff bool - ForwardEthCalls bool - ProxyOnError bool - NodeNetworkID string + ChainConfig *params.ChainConfig + DefaultSender *common.Address + RPCGasCap *big.Int + EthHttpEndpoint string + Client *rpc.Client + SupportStateDiff bool + ForwardEthCalls bool + ForwardGetStorageAt bool + ProxyOnError bool + NodeNetworkID string // Cache configuration. GroupCache *ethServerShared.GroupCacheConfig @@ -108,6 +110,7 @@ func NewConfig() (*Config, error) { viper.BindEnv("ethereum.chainConfig", ETH_CHAIN_CONFIG) viper.BindEnv("ethereum.supportsStateDiff", ETH_SUPPORTS_STATEDIFF) viper.BindEnv("ethereum.forwardEthCalls", ETH_FORWARD_ETH_CALLS) + viper.BindEnv("ethereum.forwardGetStorageAt", ETH_FORWARD_GET_STORAGE_AT) viper.BindEnv("ethereum.proxyOnError", ETH_PROXY_ON_ERROR) c.dbInit() @@ -121,6 +124,7 @@ func NewConfig() (*Config, error) { c.Client = cli c.SupportStateDiff = viper.GetBool("ethereum.supportsStateDiff") c.ForwardEthCalls = viper.GetBool("ethereum.forwardEthCalls") + c.ForwardGetStorageAt = viper.GetBool("ethereum.forwardGetStorageAt") c.ProxyOnError = viper.GetBool("ethereum.proxyOnError") c.EthHttpEndpoint = ethHTTPEndpoint diff --git a/pkg/serve/service.go b/pkg/serve/service.go index b22963cc..10d8f708 100644 --- a/pkg/serve/service.go +++ b/pkg/serve/service.go @@ -85,6 +85,8 @@ type Service struct { backend *eth.Backend // whether to forward eth_calls directly to proxy node forwardEthCalls bool + // whether to forward eth_getStorageAt directly to proxy node + forwardGetStorageAt bool // whether to forward all calls to proxy node if they throw an error locally proxyOnError bool // eth node network id @@ -104,6 +106,7 @@ func NewServer(settings *Config) (Server, error) { sap.client = settings.Client sap.supportsStateDiffing = settings.SupportStateDiff sap.forwardEthCalls = settings.ForwardEthCalls + sap.forwardGetStorageAt = settings.ForwardGetStorageAt sap.proxyOnError = settings.ProxyOnError sap.nodeNetworkId = settings.NodeNetworkID var err error @@ -139,7 +142,7 @@ func (sap *Service) APIs() []rpc.API { Public: true, }, } - ethAPI, err := eth.NewPublicEthAPI(sap.backend, sap.client, sap.supportsStateDiffing, sap.forwardEthCalls, sap.proxyOnError) + ethAPI, err := eth.NewPublicEthAPI(sap.backend, sap.client, sap.supportsStateDiffing, sap.forwardEthCalls, sap.forwardGetStorageAt, sap.proxyOnError) if err != nil { log.Fatalf("unable to create public eth api: %v", err) } From 71cbb71e136a38ea26818f73fa13bcf302eb6195 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Tue, 10 Jan 2023 12:13:12 -0600 Subject: [PATCH 3/7] Remove accidental file. --- .github/workflows/issues-notion-sync.yml | 29 ------------------------ 1 file changed, 29 deletions(-) delete mode 100644 .github/workflows/issues-notion-sync.yml diff --git a/.github/workflows/issues-notion-sync.yml b/.github/workflows/issues-notion-sync.yml deleted file mode 100644 index 551362b0..00000000 --- a/.github/workflows/issues-notion-sync.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Notion Sync - -on: - workflow_dispatch: - issues: - types: - [ - opened, - edited, - labeled, - unlabeled, - assigned, - unassigned, - milestoned, - demilestoned, - reopened, - closed, - ] - -jobs: - notion_job: - runs-on: ubuntu-latest - name: Add GitHub Issues to Notion - steps: - - name: Add GitHub Issues to Notion - uses: vulcanize/notion-github-action@v1.2.4-issueid - with: - notion-token: ${{ secrets.NOTION_TOKEN }} - notion-db: ${{ secrets.NOTION_DATABASE }} From c57e8091b899595fcfaffeaa8ec410471c5aefa8 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Tue, 10 Jan 2023 12:19:38 -0600 Subject: [PATCH 4/7] Merge error --- pkg/eth/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/eth/api.go b/pkg/eth/api.go index bc5a8091..551b0452 100644 --- a/pkg/eth/api.go +++ b/pkg/eth/api.go @@ -71,7 +71,7 @@ type PublicEthAPI struct { } // NewPublicEthAPI creates a new PublicEthAPI with the provided underlying Backend -func NewPublicEthAPI(b *Backend, client *rpc.Client, supportsStateDiff, forwardEthCalls, forwardGetStorageAt, proxyOnError bool) (*P +func NewPublicEthAPI(b *Backend, client *rpc.Client, supportsStateDiff, forwardEthCalls, forwardGetStorageAt, proxyOnError bool) (*PublicEthAPI, error) { if b == nil { return nil, errors.New("ipld-eth-server must be configured with an ethereum backend") } From 3a248979b99977d4e6a004dab1924bea335672a2 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Tue, 10 Jan 2023 12:24:29 -0600 Subject: [PATCH 5/7] Remove ipld --- pkg/eth/ipld_retriever.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/eth/ipld_retriever.go b/pkg/eth/ipld_retriever.go index 9f89f7dd..3cfb6530 100644 --- a/pkg/eth/ipld_retriever.go +++ b/pkg/eth/ipld_retriever.go @@ -29,7 +29,6 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" "github.com/lib/pq" - log "github.com/sirupsen/logrus" ) const ( @@ -668,8 +667,6 @@ func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageSlotAndBlockHash(add if err := r.db.Get(storageResult, RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr, stateLeafKey.Hex(), storageHash.Hex(), hash.Hex()); err != nil { return "", nil, nil, err } - log.Debugf("getStorageAt: state_leaf_key=%s, storage_hex=%s, storage_leaf_key=%s, block_hash=%s", - stateLeafKey.Hex(), key.Hex(), storageHash.Hex(), hash.Hex()) if storageResult.StateLeafRemoved || storageResult.NodeType == sdtypes.Removed.Int() { return "", EmptyNodeValue, EmptyNodeValue, nil } From 96a077472573a98651967f7cd2669ba65cb0d340 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Tue, 10 Jan 2023 13:16:01 -0600 Subject: [PATCH 6/7] Always return when forwarding. --- pkg/eth/api.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/eth/api.go b/pkg/eth/api.go index 551b0452..5cd26e41 100644 --- a/pkg/eth/api.go +++ b/pkg/eth/api.go @@ -733,10 +733,10 @@ func (pea *PublicEthAPI) GetStorageAt(ctx context.Context, address common.Addres if pea.forwardGetStorageAt { var res hexutil.Bytes // If forwarding all getStorageAt calls, don't request statediffing. - if err := pea.rpc.CallContext(ctx, &res, "eth_getStorageAt", address, key, blockNrOrHash); res != nil && err == nil { - return res, nil - } + err := pea.rpc.CallContext(ctx, &res, "eth_getStorageAt", address, key, blockNrOrHash) + return res, err } + storageVal, err := pea.B.GetStorageByNumberOrHash(ctx, address, common.HexToHash(key), blockNrOrHash) if storageVal != nil && err == nil { var value common.Hash From f667e59699414cf2db2512c0506826871a2344e3 Mon Sep 17 00:00:00 2001 From: i-norden Date: Tue, 10 Jan 2023 15:34:39 -0600 Subject: [PATCH 7/7] do no eth_writeStateDiffAt for certain endpoints where we cannot pinpoint which statediff is missing --- pkg/eth/api.go | 25 ++----------------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/pkg/eth/api.go b/pkg/eth/api.go index 5cd26e41..e52f4e0f 100644 --- a/pkg/eth/api.go +++ b/pkg/eth/api.go @@ -707,7 +707,6 @@ func (pea *PublicEthAPI) GetBalance(ctx context.Context, address common.Address, if pea.proxyOnError { var res *hexutil.Big if err := pea.rpc.CallContext(ctx, &res, "eth_getBalance", address, blockNrOrHash); res != nil && err == nil { - go pea.writeStateDiffAtOrFor(blockNrOrHash) return res, nil } } @@ -755,8 +754,6 @@ func (pea *PublicEthAPI) GetStorageAt(ctx context.Context, address common.Addres logrus.Warnf("Missing eth_getStorageAt(%s, %s, %s)", address.Hash().String(), key, blockNrOrHash.String()) var res hexutil.Bytes if err := pea.rpc.CallContext(ctx, &res, "eth_getStorageAt", address, key, blockNrOrHash); res != nil && err == nil { - // If only proxying on error, request statediffing for the missing data. - go pea.writeStateDiffAtOrFor(blockNrOrHash) return res, nil } } @@ -775,7 +772,6 @@ func (pea *PublicEthAPI) GetCode(ctx context.Context, address common.Address, bl if pea.proxyOnError { var res hexutil.Bytes if err := pea.rpc.CallContext(ctx, &res, "eth_getCode", address, blockNrOrHash); res != nil && err == nil { - go pea.writeStateDiffAtOrFor(blockNrOrHash) return res, nil } } @@ -795,7 +791,6 @@ func (pea *PublicEthAPI) GetProof(ctx context.Context, address common.Address, s if pea.proxyOnError { var res *AccountResult if err := pea.rpc.CallContext(ctx, &res, "eth_getProof", address, storageKeys, blockNrOrHash); res != nil && err == nil { - go pea.writeStateDiffAtOrFor(blockNrOrHash) return res, nil } } @@ -965,7 +960,6 @@ func (pea *PublicEthAPI) Call(ctx context.Context, args CallArgs, blockNrOrHash if err != nil && pea.proxyOnError { var hex hexutil.Bytes if err := pea.rpc.CallContext(ctx, &hex, "eth_call", args, blockNrOrHash, overrides); hex != nil && err == nil { - go pea.writeStateDiffAtOrFor(blockNrOrHash) return hex, nil } } @@ -1056,25 +1050,10 @@ func (pea *PublicEthAPI) writeStateDiffAtOrFor(blockNrOrHash rpc.BlockNumberOrHa // writeStateDiffWithCriteria calls out to the proxy statediffing geth client to fill in a gap in the index func (pea *PublicEthAPI) writeStateDiffWithCriteria(crit filters.FilterCriteria) { // short circuit right away if the proxy doesn't support diffing - if !pea.supportsStateDiff { + if !pea.supportsStateDiff || crit.BlockHash == nil { return } - if crit.BlockHash != nil { - pea.writeStateDiffFor(*crit.BlockHash) - return - } - var start, end int64 - if crit.FromBlock != nil { - start = crit.FromBlock.Int64() - } - if crit.ToBlock != nil { - end = crit.ToBlock.Int64() - } else { - end = start - } - for i := start; i <= end; i++ { - pea.writeStateDiffAt(i) - } + pea.writeStateDiffFor(*crit.BlockHash) } // writeStateDiffAt calls out to the proxy statediffing geth client to fill in a gap in the index