deep-stack/ipld-eth-server
4
0
Fork 0
forked from cerc-io/ipld-eth-server
Code Pull Requests Activity

composeAndExecute command that loads and executes over arbitrary transformer set exported from a go plugin generated according to config file; test for plugin generation, loading, and execution; work on plugins README

Browse Source
This commit is contained in:
Ian Norden
2019-02-24 15:23:35 -06:00
parent fa4abe4e5c
commit 6c2d895023
116 changed files with 10554 additions and 454 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libraries/shared/transformer/transformer.go
+46
View File
@@ -0,0 +1,46 @@
// Auto-gen this code for different transformer interfaces/configs
// based on config file to allow for more modularity
package transformer
import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
)
type Transformer interface {
Execute(logs []types.Log, header core.Header, recheckHeaders constants.TransformerExecution) error
GetConfig() TransformerConfig
}
type TransformerInitializer func(db *postgres.DB) Transformer
type TransformerConfig struct {
TransformerName string
ContractAddresses []string
ContractAbi string
Topic string
StartingBlockNumber int64
EndingBlockNumber int64 // Set -1 for indefinite transformer
}
func HexToInt64(byteString string) int64 {
value := common.HexToHash(byteString)
return value.Big().Int64()
}
func HexToString(byteString string) string {
value := common.HexToHash(byteString)
return value.Big().String()
}
func HexStringsToAddresses(strings []string) (addresses []common.Address) {
for _, hexString := range strings {
addresses = append(addresses, common.HexToAddress(hexString))
}
return
}
libraries/shared/transformer/transformer_suite_test.go
+35
View File
@@ -0,0 +1,35 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package transformer_test
import (
"io/ioutil"
"log"
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
func TestShared(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Shared Transformer Suite")
}
var _ = BeforeSuite(func() {
log.SetOutput(ioutil.Discard)
})
libraries/shared/transformer/transformer_test.go
+17
View File
@@ -0,0 +1,17 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package transformer_test
libraries/shared/event_watcher.go → libraries/shared/watcher/event_watcher.go
+14 -11
View File
@@ -14,12 +14,15 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package shared
package watcher
import (
"fmt"
"github.com/ethereum/go-ethereum/common"
log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
@@ -27,7 +30,7 @@ import (
)
type EventWatcher struct {
Transformers []shared.Transformer
Transformers []transformer.Transformer
DB *postgres.DB
Fetcher shared.LogFetcher
Chunker shared.Chunker
@@ -47,16 +50,16 @@ func NewEventWatcher(db *postgres.DB, bc core.BlockChain) EventWatcher {
}
// Adds transformers to the watcher and updates the chunker, so that it will consider the new transformers.
func (watcher *EventWatcher) AddTransformers(initializers []shared.TransformerInitializer) {
func (watcher *EventWatcher) AddTransformers(initializers []transformer.TransformerInitializer) {
var contractAddresses []common.Address
var topic0s []common.Hash
var configs []shared.TransformerConfig
var configs []transformer.TransformerConfig
for _, initializer := range initializers {
transformer := initializer(watcher.DB)
watcher.Transformers = append(watcher.Transformers, transformer)
t := initializer(watcher.DB)
watcher.Transformers = append(watcher.Transformers, t)
config := transformer.GetConfig()
config := t.GetConfig()
configs = append(configs, config)
if watcher.StartingBlock == nil {
@@ -65,7 +68,7 @@ func (watcher *EventWatcher) AddTransformers(initializers []shared.TransformerIn
watcher.StartingBlock = &config.StartingBlockNumber
}
addresses := shared.HexStringsToAddresses(config.ContractAddresses)
addresses := transformer.HexStringsToAddresses(config.ContractAddresses)
contractAddresses = append(contractAddresses, addresses...)
topic0s = append(topic0s, common.HexToHash(config.Topic))
}
@@ -105,10 +108,10 @@ func (watcher *EventWatcher) Execute(recheckHeaders constants.TransformerExecuti
// Can't quit early and mark as checked if there are no logs. If we are running continuousLogSync,
// not all logs we're interested in might have been fetched.
for _, transformer := range watcher.Transformers {
transformerName := transformer.GetConfig().TransformerName
for _, t := range watcher.Transformers {
transformerName := t.GetConfig().TransformerName
logChunk := chunkedLogs[transformerName]
err = transformer.Execute(logChunk, header, constants.HeaderMissing)
err = t.Execute(logChunk, header, constants.HeaderMissing)
if err != nil {
log.Errorf("%v transformer failed to execute in watcher: %v", transformerName, err)
return err
libraries/shared/event_watcher_test.go → libraries/shared/watcher/event_watcher_test.go
+44 -46
View File
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package shared_test
package watcher_test
import (
"errors"
@@ -25,85 +25,85 @@ import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/libraries/shared"
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
"github.com/vulcanize/vulcanizedb/libraries/shared/watcher"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
shared2 "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks"
"github.com/vulcanize/vulcanizedb/test_config"
)
var _ = Describe("EventWatcher", func() {
var _ = Describe("Watcher", func() {
It("initialises correctly", func() {
db := test_config.NewTestDB(core.Node{ID: "testNode"})
bc := fakes.NewMockBlockChain()
watcher := shared.NewEventWatcher(db, bc)
w := watcher.NewEventWatcher(db, bc)
Expect(watcher.DB).To(Equal(db))
Expect(watcher.Fetcher).NotTo(BeNil())
Expect(watcher.Chunker).NotTo(BeNil())
Expect(w.DB).To(Equal(db))
Expect(w.Fetcher).NotTo(BeNil())
Expect(w.Chunker).NotTo(BeNil())
})
It("adds transformers", func() {
watcher := shared.NewEventWatcher(nil, nil)
w := watcher.NewEventWatcher(nil, nil)
fakeTransformer := &mocks.MockTransformer{}
fakeTransformer.SetTransformerConfig(mocks.FakeTransformerConfig)
watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformer.FakeTransformerInitializer})
w.AddTransformers([]transformer.TransformerInitializer{fakeTransformer.FakeTransformerInitializer})
Expect(len(watcher.Transformers)).To(Equal(1))
Expect(watcher.Transformers).To(ConsistOf(fakeTransformer))
Expect(watcher.Topics).To(Equal([]common.Hash{common.HexToHash("FakeTopic")}))
Expect(watcher.Addresses).To(Equal([]common.Address{common.HexToAddress("FakeAddress")}))
Expect(len(w.Transformers)).To(Equal(1))
Expect(w.Transformers).To(ConsistOf(fakeTransformer))
Expect(w.Topics).To(Equal([]common.Hash{common.HexToHash("FakeTopic")}))
Expect(w.Addresses).To(Equal([]common.Address{common.HexToAddress("FakeAddress")}))
})
It("adds transformers from multiple sources", func() {
watcher := shared.NewEventWatcher(nil, nil)
w := watcher.NewEventWatcher(nil, nil)
fakeTransformer1 := &mocks.MockTransformer{}
fakeTransformer1.SetTransformerConfig(mocks.FakeTransformerConfig)
fakeTransformer2 := &mocks.MockTransformer{}
fakeTransformer2.SetTransformerConfig(mocks.FakeTransformerConfig)
watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformer1.FakeTransformerInitializer})
watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformer2.FakeTransformerInitializer})
w.AddTransformers([]transformer.TransformerInitializer{fakeTransformer1.FakeTransformerInitializer})
w.AddTransformers([]transformer.TransformerInitializer{fakeTransformer2.FakeTransformerInitializer})
Expect(len(watcher.Transformers)).To(Equal(2))
Expect(watcher.Topics).To(Equal([]common.Hash{common.HexToHash("FakeTopic"),
Expect(len(w.Transformers)).To(Equal(2))
Expect(w.Topics).To(Equal([]common.Hash{common.HexToHash("FakeTopic"),
common.HexToHash("FakeTopic")}))
Expect(watcher.Addresses).To(Equal([]common.Address{common.HexToAddress("FakeAddress"),
Expect(w.Addresses).To(Equal([]common.Address{common.HexToAddress("FakeAddress"),
common.HexToAddress("FakeAddress")}))
})
It("calculates earliest starting block number", func() {
fakeTransformer1 := &mocks.MockTransformer{}
fakeTransformer1.SetTransformerConfig(shared2.TransformerConfig{StartingBlockNumber: 5})
fakeTransformer1.SetTransformerConfig(transformer.TransformerConfig{StartingBlockNumber: 5})
fakeTransformer2 := &mocks.MockTransformer{}
fakeTransformer2.SetTransformerConfig(shared2.TransformerConfig{StartingBlockNumber: 3})
fakeTransformer2.SetTransformerConfig(transformer.TransformerConfig{StartingBlockNumber: 3})
watcher := shared.NewEventWatcher(nil, nil)
watcher.AddTransformers([]shared2.TransformerInitializer{
w := watcher.NewEventWatcher(nil, nil)
w.AddTransformers([]transformer.TransformerInitializer{
fakeTransformer1.FakeTransformerInitializer,
fakeTransformer2.FakeTransformerInitializer,
})
Expect(*watcher.StartingBlock).To(Equal(int64(3)))
Expect(*w.StartingBlock).To(Equal(int64(3)))
})
It("returns an error when run without transformers", func() {
watcher := shared.NewEventWatcher(nil, nil)
err := watcher.Execute(constants.HeaderMissing)
w := watcher.NewEventWatcher(nil, nil)
err := w.Execute(constants.HeaderMissing)
Expect(err).To(MatchError("No transformers added to watcher"))
})
Describe("with missing headers", func() {
var (
db *postgres.DB
watcher shared.EventWatcher
w watcher.EventWatcher
mockBlockChain fakes.MockBlockChain
headerRepository repositories.HeaderRepository
repository mocks.MockWatcherRepository
@@ -118,27 +118,25 @@ var _ = Describe("EventWatcher", func() {
Expect(err).NotTo(HaveOccurred())
repository = mocks.MockWatcherRepository{}
watcher = shared.NewEventWatcher(db, &mockBlockChain)
w = watcher.NewEventWatcher(db, &mockBlockChain)
})
It("executes each transformer", func() {
fakeTransformer := &mocks.MockTransformer{}
watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformer.FakeTransformerInitializer})
w.AddTransformers([]transformer.TransformerInitializer{fakeTransformer.FakeTransformerInitializer})
repository.SetMissingHeaders([]core.Header{fakes.FakeHeader})
err := watcher.Execute(constants.HeaderMissing)
err := w.Execute(constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred())
Expect(fakeTransformer.ExecuteWasCalled).To(BeTrue())
})
It("returns an error if transformer returns an error", func() {
fakeTransformer := &mocks.MockTransformer{ExecuteError: errors.New("Something bad happened")}
watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformer.FakeTransformerInitializer})
w.AddTransformers([]transformer.TransformerInitializer{fakeTransformer.FakeTransformerInitializer})
repository.SetMissingHeaders([]core.Header{fakes.FakeHeader})
err := watcher.Execute(constants.HeaderMissing)
err := w.Execute(constants.HeaderMissing)
Expect(err).To(HaveOccurred())
Expect(fakeTransformer.ExecuteWasCalled).To(BeFalse())
})
@@ -147,10 +145,10 @@ var _ = Describe("EventWatcher", func() {
transformerA := &mocks.MockTransformer{}
transformerB := &mocks.MockTransformer{}
configA := shared2.TransformerConfig{TransformerName: "transformerA",
configA := transformer.TransformerConfig{TransformerName: "transformerA",
ContractAddresses: []string{"0x000000000000000000000000000000000000000A"},
Topic: "0xA"}
configB := shared2.TransformerConfig{TransformerName: "transformerB",
configB := transformer.TransformerConfig{TransformerName: "transformerB",
ContractAddresses: []string{"0x000000000000000000000000000000000000000b"},
Topic: "0xB"}
@@ -164,11 +162,11 @@ var _ = Describe("EventWatcher", func() {
mockBlockChain.SetGetEthLogsWithCustomQueryReturnLogs([]types.Log{logA, logB})
repository.SetMissingHeaders([]core.Header{fakes.FakeHeader})
watcher = shared.NewEventWatcher(db, &mockBlockChain)
watcher.AddTransformers([]shared2.TransformerInitializer{
w = watcher.NewEventWatcher(db, &mockBlockChain)
w.AddTransformers([]transformer.TransformerInitializer{
transformerA.FakeTransformerInitializer, transformerB.FakeTransformerInitializer})
err := watcher.Execute(constants.HeaderMissing)
err := w.Execute(constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred())
Expect(transformerA.PassedLogs).To(Equal([]types.Log{logA}))
Expect(transformerB.PassedLogs).To(Equal([]types.Log{logB}))
@@ -184,17 +182,17 @@ var _ = Describe("EventWatcher", func() {
It("fetches logs for added transformers", func() {
addresses := []string{"0xA", "0xB"}
topic := "0x1"
fakeTransformer.SetTransformerConfig(shared2.TransformerConfig{
fakeTransformer.SetTransformerConfig(transformer.TransformerConfig{
Topic: topic, ContractAddresses: addresses})
watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformer.FakeTransformerInitializer})
w.AddTransformers([]transformer.TransformerInitializer{fakeTransformer.FakeTransformerInitializer})
err := watcher.Execute(constants.HeaderMissing)
err := w.Execute(constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred())
fakeHash := common.HexToHash(fakes.FakeHeader.Hash)
mockBlockChain.AssertGetEthLogsWithCustomQueryCalledWith(ethereum.FilterQuery{
BlockHash: &fakeHash,
Addresses: shared2.HexStringsToAddresses(addresses),
Addresses: transformer.HexStringsToAddresses(addresses),
Topics: [][]common.Hash{{common.HexToHash(topic)}},
})
})
@@ -203,8 +201,8 @@ var _ = Describe("EventWatcher", func() {
fetcherError := errors.New("FetcherError")
mockBlockChain.SetGetEthLogsWithCustomQueryErr(fetcherError)
watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformer.FakeTransformerInitializer})
err := watcher.Execute(constants.HeaderMissing)
w.AddTransformers([]transformer.TransformerInitializer{fakeTransformer.FakeTransformerInitializer})
err := w.Execute(constants.HeaderMissing)
Expect(err).To(MatchError(fetcherError))
})
})
libraries/shared/shared_suite_test.go → libraries/shared/watcher/watcher_suite_test.go
+2 -2
View File
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package shared_test
package watcher_test
import (
log "github.com/sirupsen/logrus"
@@ -27,7 +27,7 @@ import (
func TestShared(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Shared Suite")
RunSpecs(t, "Shared Watcher Suite")
}
var _ = BeforeSuite(func() {