From 7e5e12f488b2218ded5dd5b39dfa84a9b14f2329 Mon Sep 17 00:00:00 2001 From: Matt K <1036969+mkrump@users.noreply.github.com> Date: Fri, 22 Dec 2017 11:42:35 -0600 Subject: [PATCH] Backfill/listen for contract logs (#113) --- Gododir/main.go | 4 +- README.md | 4 +- cmd/get_logs/main.go | 64 ++++++++++++++++++++++--------- integration_test/contract_test.go | 4 +- pkg/core/blockchain.go | 2 +- pkg/fakes/blockchain.go | 2 +- pkg/geth/geth_blockchain.go | 10 ++--- 7 files changed, 57 insertions(+), 33 deletions(-) diff --git a/Gododir/main.go b/Gododir/main.go index ee2a4406..315c55d5 100644 --- a/Gododir/main.go +++ b/Gododir/main.go @@ -44,16 +44,14 @@ func tasks(p *do.Project) { p.Task("getLogs", nil, func(context *do.Context) { environment := parseEnvironment(context) - blockNumber := context.Args.MayInt(-1, "block-number", "b") contractHash := context.Args.MayString("", "contract-hash", "c") if contractHash == "" { log.Fatalln("--contract-hash required") } - context.Start(`go run main.go --environment={{.environment}} --contract-hash={{.contractHash}} --block-number={{.blockNumber}}`, + context.Start(`go run main.go --environment={{.environment}} --contract-hash={{.contractHash}}`, do.M{ "environment": environment, "contractHash": contractHash, - "blockNumber": blockNumber, "$in": "cmd/get_logs", }) }) diff --git a/README.md b/README.md index d3fb6f48..5af21eaa 100644 --- a/README.md +++ b/README.md @@ -93,8 +93,8 @@ The name of the JSON file should correspond the contract's address. ## Retrieving Contract Logs -1. Get the logs - - `godo getLogs -- --environment= --contract-hash= --starting-number=` +1. Get the logs for a specific contract + - `godo getLogs -- --environment= --contract-hash=` ### Configuring Additional Environments diff --git a/cmd/get_logs/main.go b/cmd/get_logs/main.go index cf9c8140..e7302a06 100644 --- a/cmd/get_logs/main.go +++ b/cmd/get_logs/main.go @@ -1,45 +1,71 @@ package main import ( - "fmt" "log" "flag" + "math/big" + + "time" + "github.com/8thlight/vulcanizedb/cmd" "github.com/8thlight/vulcanizedb/pkg/core" "github.com/8thlight/vulcanizedb/pkg/geth" ) +func min(a, b int64) int64 { + if a < b { + return a + } + return b +} + +const ( + windowSize = 24 + pollingInterval = 10 * time.Second +) + func main() { environment := flag.String("environment", "", "Environment name") contractHash := flag.String("contract-hash", "", "Contract hash to show summary") - _blockNumber := flag.Int64("block-number", -1, "Block number of summary") + ticker := time.NewTicker(pollingInterval) + defer ticker.Stop() flag.Parse() config := cmd.LoadConfig(*environment) blockchain := geth.NewGethBlockchain(config.Client.IPCPath) repository := cmd.LoadPostgres(config.Database, blockchain.Node()) - blockNumber := cmd.RequestedBlockNumber(_blockNumber) - logs, err := blockchain.GetLogs(core.Contract{Hash: *contractHash}, blockNumber) - if err != nil { - log.Fatalln(err) - } - repository.CreateLogs(logs) - for _, l := range logs { - fmt.Println("\tAddress: ", l.Address) - fmt.Println("\tTxHash: ", l.TxHash) - fmt.Println("\tBlockNumber ", l.BlockNumber) - fmt.Println("\tIndex ", l.Index) - fmt.Println("\tTopics: ") - for i, topic := range l.Topics { - fmt.Printf("\t\tTopic %d: %s\n", i, topic) + lastBlockNumber := blockchain.LastBlock().Int64() + stepSize := int64(1000) + + go func() { + for i := int64(0); i < lastBlockNumber; i = min(i+stepSize, lastBlockNumber) { + logs, err := blockchain.GetLogs(core.Contract{Hash: *contractHash}, big.NewInt(i), big.NewInt(i+stepSize)) + log.Println("Backfilling Logs:", i) + if err != nil { + log.Println(err) + } + repository.CreateLogs(logs) } - fmt.Printf("\tData: %s", l.Data) - fmt.Print("\n\n") + }() + done := make(chan struct{}) + go func() { done <- struct{}{} }() + for range ticker.C { + select { + case <-done: + go func() { + z := &big.Int{} + z.Sub(blockchain.LastBlock(), big.NewInt(25)) + log.Printf("Logs Window: %d - %d", z.Int64(), blockchain.LastBlock().Int64()) + logs, _ := blockchain.GetLogs(core.Contract{Hash: *contractHash}, z, blockchain.LastBlock()) + repository.CreateLogs(logs) + done <- struct{}{} + }() + default: + } } - } diff --git a/integration_test/contract_test.go b/integration_test/contract_test.go index c8377320..24588c19 100644 --- a/integration_test/contract_test.go +++ b/integration_test/contract_test.go @@ -119,7 +119,7 @@ var _ = Describe("Reading contracts", func() { blockchain := geth.NewGethBlockchain(config.Client.IPCPath) contract := testing.SampleContract() - logs, err := blockchain.GetLogs(contract, big.NewInt(4703824)) + logs, err := blockchain.GetLogs(contract, big.NewInt(4703824), nil) Expect(err).To(BeNil()) Expect(len(logs)).To(Equal(3)) @@ -131,7 +131,7 @@ var _ = Describe("Reading contracts", func() { config, _ := cfg.NewConfig("infura") blockchain := geth.NewGethBlockchain(config.Client.IPCPath) - logs, err := blockchain.GetLogs(core.Contract{Hash: "x123"}, big.NewInt(4703824)) + logs, err := blockchain.GetLogs(core.Contract{Hash: "x123"}, big.NewInt(4703824), nil) Expect(err).To(BeNil()) Expect(len(logs)).To(Equal(0)) diff --git a/pkg/core/blockchain.go b/pkg/core/blockchain.go index 6207a4ae..e5f1c0af 100644 --- a/pkg/core/blockchain.go +++ b/pkg/core/blockchain.go @@ -11,5 +11,5 @@ type Blockchain interface { StopListening() GetAttributes(contract Contract) (ContractAttributes, error) GetAttribute(contract Contract, attributeName string, blockNumber *big.Int) (interface{}, error) - GetLogs(contract Contract, blockNumber *big.Int) ([]Log, error) + GetLogs(contract Contract, startingBlockNumber *big.Int, endingBlockNumber *big.Int) ([]Log, error) } diff --git a/pkg/fakes/blockchain.go b/pkg/fakes/blockchain.go index 6cda5569..0c4b1f73 100644 --- a/pkg/fakes/blockchain.go +++ b/pkg/fakes/blockchain.go @@ -27,7 +27,7 @@ func (blockchain *Blockchain) LastBlock() *big.Int { return big.NewInt(max) } -func (blockchain *Blockchain) GetLogs(contract core.Contract, blockNumber *big.Int) ([]core.Log, error) { +func (blockchain *Blockchain) GetLogs(contract core.Contract, startingBlock *big.Int, endingBlock *big.Int) ([]core.Log, error) { return blockchain.logs[contract.Hash], nil } diff --git a/pkg/geth/geth_blockchain.go b/pkg/geth/geth_blockchain.go index 2be4d378..802d48ea 100644 --- a/pkg/geth/geth_blockchain.go +++ b/pkg/geth/geth_blockchain.go @@ -23,14 +23,14 @@ type GethBlockchain struct { node core.Node } -func (blockchain *GethBlockchain) GetLogs(contract core.Contract, blockNumber *big.Int) ([]core.Log, error) { - if blockNumber == nil { - blockNumber = blockchain.LastBlock() +func (blockchain *GethBlockchain) GetLogs(contract core.Contract, startingBlockNumber *big.Int, endingBlockNumber *big.Int) ([]core.Log, error) { + if endingBlockNumber == nil { + endingBlockNumber = startingBlockNumber } contractAddress := common.HexToAddress(contract.Hash) fc := ethereum.FilterQuery{ - FromBlock: blockNumber, - ToBlock: blockNumber, + FromBlock: startingBlockNumber, + ToBlock: endingBlockNumber, Addresses: []common.Address{contractAddress}, } gethLogs, err := blockchain.client.FilterLogs(context.Background(), fc)