cerc-io/ipld-eth-server
18
0
Fork 1
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases 37 Wiki Activity

Merge pull request #128 from 8thlight/VDB-125-aggregate-log-fetching

Browse Source 
VDB-125 Aggregate log fetching
This commit is contained in:
Edvard Hübinette 2019-01-08 10:51:42 +01:00 committed by GitHub
commit 77209d3b62
120 changed files with 1642 additions and 2574 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
cmd/backfillMakerLogs.go
View File

@ -17,7 +17,6 @@ package cmd
import (
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/vulcanize/vulcanizedb/libraries/shared"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers"
@ -46,13 +45,13 @@ func backfillMakerLogs() {
log.Fatal("Failed to initialize database.")
}
watcher := shared.Watcher{
DB: *db,
Blockchain: blockChain,
}
watcher := shared.NewWatcher(db, blockChain)
watcher.AddTransformers(transformers.TransformerInitializers())
watcher.Execute()
err = watcher.Execute()
if err != nil {
// TODO Handle watcher error in backfillMakerLogs
}
}
func init() {

90
cmd/continuousLogSync.go
View File

@ -58,70 +58,66 @@ func syncMakerLogs() {
log.Fatal("Failed to initialize database.")
}
watcher := shared.Watcher{
DB: *db,
Blockchain: blockChain,
}
initializers := getTransformerInitializers(transformerNames)
transformerInititalizers := getTransformerInititalizers(transformerNames)
watcher.AddTransformers(transformerInititalizers)
watcher := shared.NewWatcher(db, blockChain)
watcher.AddTransformers(initializers)
for range ticker.C {
watcher.Execute()
err = watcher.Execute()
if err != nil {
// TODO Handle watcher errors in ContinuousLogSync
}
}
}
func getTransformerInititalizers(transformerNames []string) []shared2.TransformerInitializer {
transformerInitializerMap := buildTransformerInitializerMap()
var transformerInitializers []shared2.TransformerInitializer
func getTransformerInitializers(transformerNames []string) []shared2.TransformerInitializer {
var initializers []shared2.TransformerInitializer
if transformerNames[0] == "all" {
for _, v := range transformerInitializerMap {
transformerInitializers = append(transformerInitializers, v)
}
initializers = transformers.TransformerInitializers()
} else {
initializerMap := buildTransformerInitializerMap()
for _, transformerName := range transformerNames {
initializer := transformerInitializerMap[transformerName]
transformerInitializers = append(transformerInitializers, initializer)
initializers = append(initializers, initializerMap[transformerName])
}
}
return transformerInitializers
return initializers
}
func buildTransformerInitializerMap() map[string]shared2.TransformerInitializer {
transformerInitializerMap := make(map[string]shared2.TransformerInitializer)
initializerMap := make(map[string]shared2.TransformerInitializer)
transformerInitializerMap[constants.BiteLabel] = transformers.BiteTransformerInitializer
transformerInitializerMap[constants.CatFileChopLumpLabel] = transformers.CatFileChopLumpTransformerInitializer
transformerInitializerMap[constants.CatFileFlipLabel] = transformers.CatFileFlipTransformerInitializer
transformerInitializerMap[constants.CatFilePitVowLabel] = transformers.CatFilePitVowTransformerInitializer
transformerInitializerMap[constants.DealLabel] = transformers.DealTransformerInitializer
transformerInitializerMap[constants.DentLabel] = transformers.DentTransformerInitializer
transformerInitializerMap[constants.DripDripLabel] = transformers.DripDripTransformerInitializer
transformerInitializerMap[constants.DripFileIlkLabel] = transformers.DripFileIlkTransformerInitializer
transformerInitializerMap[constants.DripFileRepoLabel] = transformers.DripFileRepoTransformerInitializer
transformerInitializerMap[constants.DripFileVowLabel] = transformers.DripFileVowTransfromerInitializer
transformerInitializerMap[constants.FlapKickLabel] = transformers.FlapKickTransformerInitializer
transformerInitializerMap[constants.FlipKickLabel] = transformers.FlipKickTransformerInitializer
transformerInitializerMap[constants.FlopKickLabel] = transformers.FlopKickTransformerInitializer
transformerInitializerMap[constants.FrobLabel] = transformers.FrobTransformerInitializer
transformerInitializerMap[constants.PitFileDebtCeilingLabel] = transformers.PitFileDebtCeilingTransformerInitializer
transformerInitializerMap[constants.PitFileIlkLabel] = transformers.PitFileIlkTransformerInitializer
transformerInitializerMap[constants.PriceFeedLabel] = transformers.PriceFeedTransformerInitializer
transformerInitializerMap[constants.TendLabel] = transformers.TendTransformerInitializer
transformerInitializerMap[constants.VatFluxLabel] = transformers.VatFluxTransformerInitializer
transformerInitializerMap[constants.VatFoldLabel] = transformers.VatFoldTransformerInitializer
transformerInitializerMap[constants.VatGrabLabel] = transformers.VatGrabTransformerInitializer
transformerInitializerMap[constants.VatHealLabel] = transformers.VatHealTransformerInitializer
transformerInitializerMap[constants.VatInitLabel] = transformers.VatInitTransformerInitializer
transformerInitializerMap[constants.VatMoveLabel] = transformers.VatMoveTransformerInitializer
transformerInitializerMap[constants.VatSlipLabel] = transformers.VatSlipTransformerInitializer
transformerInitializerMap[constants.VatTollLabel] = transformers.VatTollTransformerInitializer
transformerInitializerMap[constants.VatTuneLabel] = transformers.VatTuneTransformerInitializer
transformerInitializerMap[constants.VowFlogLabel] = transformers.FlogTransformerInitializer
initializerMap[constants.BiteLabel] = transformers.BiteTransformer.NewTransformer
initializerMap[constants.CatFileChopLumpLabel] = transformers.CatFileChopLumpTransformer.NewLogNoteTransformer
initializerMap[constants.CatFileFlipLabel] = transformers.CatFileFlipTransformer.NewLogNoteTransformer
initializerMap[constants.CatFilePitVowLabel] = transformers.CatFilePitVowTransformer.NewLogNoteTransformer
initializerMap[constants.DealLabel] = transformers.DealTransformer.NewLogNoteTransformer
initializerMap[constants.DentLabel] = transformers.DentTransformer.NewLogNoteTransformer
initializerMap[constants.DripDripLabel] = transformers.DripDripTransformer.NewLogNoteTransformer
initializerMap[constants.DripFileIlkLabel] = transformers.DripFileIlkTransformer.NewLogNoteTransformer
initializerMap[constants.DripFileRepoLabel] = transformers.DripFileRepoTransformer.NewLogNoteTransformer
initializerMap[constants.DripFileVowLabel] = transformers.DripFileVowTransfromer.NewLogNoteTransformer
initializerMap[constants.FlapKickLabel] = transformers.FlapKickTransformer.NewTransformer
initializerMap[constants.FlipKickLabel] = transformers.FlipKickTransformer.NewTransformer
initializerMap[constants.FlopKickLabel] = transformers.FlopKickTransformer.NewTransformer
initializerMap[constants.FrobLabel] = transformers.FrobTransformer.NewTransformer
initializerMap[constants.PitFileDebtCeilingLabel] = transformers.PitFileDebtCeilingTransformer.NewLogNoteTransformer
initializerMap[constants.PitFileIlkLabel] = transformers.PitFileIlkTransformer.NewLogNoteTransformer
initializerMap[constants.PriceFeedLabel] = transformers.PriceFeedTransformer.NewLogNoteTransformer
initializerMap[constants.TendLabel] = transformers.TendTransformer.NewLogNoteTransformer
initializerMap[constants.VatFluxLabel] = transformers.VatFluxTransformer.NewLogNoteTransformer
initializerMap[constants.VatFoldLabel] = transformers.VatFoldTransformer.NewLogNoteTransformer
initializerMap[constants.VatGrabLabel] = transformers.VatGrabTransformer.NewLogNoteTransformer
initializerMap[constants.VatHealLabel] = transformers.VatHealTransformer.NewLogNoteTransformer
initializerMap[constants.VatInitLabel] = transformers.VatInitTransformer.NewLogNoteTransformer
initializerMap[constants.VatMoveLabel] = transformers.VatMoveTransformer.NewLogNoteTransformer
initializerMap[constants.VatSlipLabel] = transformers.VatSlipTransformer.NewLogNoteTransformer
initializerMap[constants.VatTollLabel] = transformers.VatTollTransformer.NewLogNoteTransformer
initializerMap[constants.VatTuneLabel] = transformers.VatTuneTransformer.NewLogNoteTransformer
initializerMap[constants.VowFlogLabel] = transformers.FlogTransformer.NewLogNoteTransformer
return transformerInitializerMap
return initializerMap
}
func init() {

71
cmd/erc20.go
View File

@ -1,71 +0,0 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cmd
import (
"time"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block"
"github.com/vulcanize/vulcanizedb/libraries/shared"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
// erc20Cmd represents the erc20 command
var erc20Cmd = &cobra.Command{
Use: "erc20",
Short: "Fetches and persists token supply",
Long: `Fetches the totalSupply for the configured token from each block and persists it in Vulcanize DB.
vulcanizedb erc20 --config environments/public
Expects an ethereum node to be running and requires a .toml config file:
[database]
name = "vulcanize_public"
hostname = "localhost"
port = 5432
[client]
ipcPath = "/Users/user/Library/Ethereum/geth.ipc"
`,
Run: func(cmd *cobra.Command, args []string) {
watchERC20s()
},
}
func watchERC20s() {
ticker := time.NewTicker(5 * time.Second)
defer ticker.Stop()
blockChain := getBlockChain()
db, err := postgres.NewDB(databaseConfig, blockChain.Node())
if err != nil {
log.Fatal("Failed to initialize database.")
}
watcher := shared.Watcher{
DB: *db,
Blockchain: blockChain,
}
watcher.AddTransformers(every_block.TransformerInitializers())
for range ticker.C {
watcher.Execute()
}
}
func init() {
rootCmd.AddCommand(erc20Cmd)
}

21
examples/constants/constants.go
View File

File diff suppressed because one or more lines are too long

27
examples/erc20_test_helpers/database.go
View File

@ -1,27 +0,0 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package erc20_test_helpers
type TokenSupplyDBRow struct {
ID int64
Supply int64
BlockID int64 `db:"block_id"`
TokenAddress string `db:"token_address"`
}
type TransferDBRow struct {
ID int64 `db:"id"`
VulcanizeLogID int64 `db:"vulcanize_log_id"`
}

106
examples/erc20_test_helpers/mocks/mocks.go
View File

@ -1,106 +0,0 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package mocks
import (
"math/big"
"github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
)
type Fetcher struct {
ContractAddress string
Abi string
FetchedBlocks []int64
BlockChain core.BlockChain
supply big.Int
}
func (f *Fetcher) SetSupply(supply string) {
f.supply.SetString(supply, 10)
}
func (f Fetcher) GetBlockChain() core.BlockChain {
return f.BlockChain
}
func (f *Fetcher) FetchSupplyOf(contractAbi string, contractAddress string, blockNumber int64) (big.Int, error) {
f.Abi = contractAbi
f.ContractAddress = contractAddress
f.FetchedBlocks = append(f.FetchedBlocks, blockNumber)
accumulator := big.NewInt(1)
f.supply.Add(&f.supply, accumulator)
return f.supply, nil
}
type TotalSupplyRepository struct {
TotalSuppliesCreated []every_block.TokenSupply
MissingBlockNumbers []int64
StartingBlock int64
EndingBlock int64
}
func (fr *TotalSupplyRepository) Create(supply every_block.TokenSupply) error {
fr.TotalSuppliesCreated = append(fr.TotalSuppliesCreated, supply)
return nil
}
func (fr *TotalSupplyRepository) MissingBlocks(startingBlock int64, highestBlock int64) ([]int64, error) {
fr.StartingBlock = startingBlock
fr.EndingBlock = highestBlock
return fr.MissingBlockNumbers, nil
}
func (fr *TotalSupplyRepository) SetMissingBlocks(missingBlocks []int64) {
fr.MissingBlockNumbers = missingBlocks
}
type FailureRepository struct {
createFail bool
missingBlocksFail bool
missingBlocksNumbers []int64
}
func (fr *FailureRepository) Create(supply every_block.TokenSupply) error {
if fr.createFail {
return fakes.FakeError
} else {
return nil
}
}
func (fr *FailureRepository) MissingBlocks(startingBlock int64, highestBlock int64) ([]int64, error) {
if fr.missingBlocksFail {
return []int64{}, fakes.FakeError
} else {
return fr.missingBlocksNumbers, nil
}
}
func (fr *FailureRepository) SetCreateFail(fail bool) {
fr.createFail = fail
}
func (fr *FailureRepository) SetMissingBlocksFail(fail bool) {
fr.missingBlocksFail = fail
}
func (fr *FailureRepository) SetMissingBlocks(missingBlocks []int64) {
fr.missingBlocksNumbers = missingBlocks
}

19
examples/erc20_watcher/ERC20WatcherREADME.md
View File

@ -1,19 +0,0 @@
# ERC20 Transformers
## Description
The Transformers in this directory are associated with contract functions and events that conform to the [ERC20 Token interface](https://theethereum.wiki/w/index.php/ERC20_Token_Standard#The_ERC20_Token_Standard_Interface).
See `libraries/shared/TransformerREADME.md` for further information regarding the Transformer interface.
## Configuration
In addition to environment configuration mentioned in the main VulcanizeDB README, the ERC20 transformers also need to be configured with contract information for the desired token(s) to be watched. This configuration file is located at `./vulcanizedb/examples/erc20_watcher/config.go`.
## ERC20 Functions
The `everyblock` directory contains transformers that fetch data from the contract itself, via one of the standard functions.
Currently, the `totalSupply` function transformer has been implemented. This transformer will fetch the total supply for the given contract address and persist `total_supply` records in the database.
## Running the transformers
1. If running against a local node, make sure that the node has been started.
1. In a separate terminal run the following command:
`./vulcanizedb erc20 --config <config.toml>`

33
examples/erc20_watcher/config.go
View File

@ -1,33 +0,0 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package erc20_watcher
import "github.com/vulcanize/vulcanizedb/examples/constants"
type ContractConfig struct {
Address string
Abi string
FirstBlock int64
LastBlock int64
Name string
}
var DaiConfig = ContractConfig{
Address: constants.DaiContractAddress,
Abi: constants.DaiAbiString,
FirstBlock: int64(4752008),
LastBlock: -1,
Name: "Dai",
}

33
examples/erc20_watcher/every_block/every_block_suite_test.go
View File

@ -1,33 +0,0 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package every_block_test
import (
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
log "github.com/sirupsen/logrus"
"io/ioutil"
)
func TestEveryBlock(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "EveryBlock Suite")
}
var _ = BeforeSuite(func() {
log.SetOutput(ioutil.Discard)
})

71
examples/erc20_watcher/every_block/fetcher.go
View File

@ -1,71 +0,0 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package every_block
import (
"fmt"
log "github.com/sirupsen/logrus"
"math/big"
"github.com/vulcanize/vulcanizedb/pkg/core"
)
type ERC20FetcherInterface interface {
FetchSupplyOf(contractAbi string, contractAddress string, blockNumber int64) (big.Int, error)
GetBlockChain() core.BlockChain
}
func NewFetcher(blockchain core.BlockChain) Fetcher {
return Fetcher{
Blockchain: blockchain,
}
}
type Fetcher struct {
Blockchain core.BlockChain
ContractAbi string
ContractAddress string
}
type fetcherError struct {
err string
fetchMethod string
}
func (fe *fetcherError) Error() string {
return fmt.Sprintf("Error fetching %s: %s", fe.fetchMethod, fe.err)
}
func newFetcherError(err error, fetchMethod string) *fetcherError {
e := fetcherError{err.Error(), fetchMethod}
log.Info(e.Error())
return &e
}
func (f Fetcher) FetchSupplyOf(contractAbi string, contractAddress string, blockNumber int64) (big.Int, error) {
method := "totalSupply"
var result = new(big.Int)
err := f.Blockchain.FetchContractData(contractAbi, contractAddress, method, nil, &result, blockNumber)
if err != nil {
return *result, newFetcherError(err, method)
}
return *result, nil
}
func (f Fetcher) GetBlockChain() core.BlockChain {
return f.Blockchain
}

82
examples/erc20_watcher/every_block/fetcher_test.go
View File

@ -1,82 +0,0 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package every_block_test
import (
"math/big"
"github.com/ethereum/go-ethereum/ethclient"
"github.com/ethereum/go-ethereum/rpc"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/examples/constants"
"github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/geth"
"github.com/vulcanize/vulcanizedb/pkg/geth/client"
rpc2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
"github.com/vulcanize/vulcanizedb/pkg/geth/node"
)
var _ = Describe("ERC20 Fetcher", func() {
blockNumber := int64(5502914)
Describe("FetchSupplyOf", func() {
It("fetches data from the blockchain with the correct arguments", func() {
fakeBlockchain := fakes.NewMockBlockChain()
testFetcher := every_block.NewFetcher(fakeBlockchain)
testAbi := "testAbi"
testContractAddress := "testContractAddress"
_, err := testFetcher.FetchSupplyOf(testAbi, testContractAddress, blockNumber)
Expect(err).NotTo(HaveOccurred())
expectedResult := big.Int{}
expected := &expectedResult
fakeBlockchain.AssertFetchContractDataCalledWith(testAbi, testContractAddress, "totalSupply", nil, &expected, blockNumber)
})
It("fetches a token's total supply at the given block height", func() {
infuraIPC := "https://mainnet.infura.io/J5Vd2fRtGsw0zZ0Ov3BL"
rawRpcClient, err := rpc.Dial(infuraIPC)
Expect(err).NotTo(HaveOccurred())
rpcClient := client.NewRpcClient(rawRpcClient, infuraIPC)
ethClient := ethclient.NewClient(rawRpcClient)
blockChainClient := client.NewEthClient(ethClient)
node := node.MakeNode(rpcClient)
transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
realFetcher := every_block.NewFetcher(blockChain)
result, err := realFetcher.FetchSupplyOf(constants.DaiAbiString, constants.DaiContractAddress, blockNumber)
Expect(err).NotTo(HaveOccurred())
expectedResult := big.Int{}
expectedResult.SetString("27647235749155415536952630", 10)
Expect(result).To(Equal(expectedResult))
})
It("returns an error if the call to the blockchain fails", func() {
blockChain := fakes.NewMockBlockChain()
blockChain.SetFetchContractDataErr(fakes.FakeError)
errorFetcher := every_block.NewFetcher(blockChain)
result, err := errorFetcher.FetchSupplyOf("", "", 0)
Expect(result.String()).To(Equal("0"))
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("totalSupply"))
Expect(err.Error()).To(ContainSubstring(fakes.FakeError.Error()))
})
})
})

83
examples/erc20_watcher/every_block/integration_test.go
View File

@ -1,83 +0,0 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package every_block_test
import (
"math/big"
"strconv"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/examples/constants"
"github.com/vulcanize/vulcanizedb/examples/erc20_test_helpers"
"github.com/vulcanize/vulcanizedb/examples/erc20_watcher"
"github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/test_config"
)
func setLastBlockOnChain(blockChain *fakes.MockBlockChain, blockNumber int64) {
blockNumberString := strconv.FormatInt(blockNumber, 10)
lastBlockOnChain := big.Int{}
lastBlockOnChain.SetString(blockNumberString, 10)
blockChain.SetLastBlock(&lastBlockOnChain)
}
var _ = Describe("Everyblock transformers", func() {
var db *postgres.DB
var blockChain *fakes.MockBlockChain
var blockNumber int64
var blockId int64
var err error
BeforeEach(func() {
blockChain = fakes.NewMockBlockChain()
blockNumber = erc20_watcher.DaiConfig.FirstBlock
lastBlockNumber := blockNumber + 1
node := test_config.NewTestNode()
db = test_config.NewTestDB(node)
test_config.CleanTestDB(db)
setLastBlockOnChain(blockChain, lastBlockNumber)
blockRepository := repositories.NewBlockRepository(db)
blockId, err = blockRepository.CreateOrUpdateBlock(core.Block{Number: blockNumber})
Expect(err).NotTo(HaveOccurred())
_, err = blockRepository.CreateOrUpdateBlock(core.Block{Number: lastBlockNumber})
Expect(err).NotTo(HaveOccurred())
})
It("creates a token_supply record for each block in the given range", func() {
initializer := every_block.TokenSupplyTransformerInitializer{Config: erc20_watcher.DaiConfig}
transformer := initializer.NewTokenSupplyTransformer(db, blockChain)
transformer.Execute()
var tokenSupplyCount int
err := db.QueryRow(`SELECT COUNT(*) FROM token_supply where block_id = $1`, blockId).Scan(&tokenSupplyCount)
Expect(err).ToNot(HaveOccurred())
Expect(tokenSupplyCount).To(Equal(1))
var tokenSupply erc20_test_helpers.TokenSupplyDBRow
err = db.Get(&tokenSupply, `SELECT * from token_supply where block_id = $1`, blockId)
Expect(err).ToNot(HaveOccurred())
Expect(tokenSupply.BlockID).To(Equal(blockId))
Expect(tokenSupply.TokenAddress).To(Equal(constants.DaiContractAddress))
Expect(tokenSupply.Supply).To(Equal(int64(0)))
})
})

21
examples/erc20_watcher/every_block/model.go
View File

@ -1,21 +0,0 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package every_block
type TokenSupply struct {
Value string
TokenAddress string
BlockNumber int64
}

91
examples/erc20_watcher/every_block/repository.go
View File

@ -1,91 +0,0 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package every_block
import (
"fmt"
log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
type ERC20RepositoryInterface interface {
Create(supply TokenSupply) error
MissingBlocks(startingBlock int64, highestBlock int64) ([]int64, error)
}
type TokenSupplyRepository struct {
*postgres.DB
}
type repositoryError struct {
err string
msg string
blockNumber int64
}
func (re *repositoryError) Error() string {
return fmt.Sprintf(re.msg, re.blockNumber, re.err)
}
func newRepositoryError(err error, msg string, blockNumber int64) error {
e := repositoryError{err.Error(), msg, blockNumber}
log.Println(e.Error())
return &e
}
const (
GetBlockError = "Error fetching block number %d: %s"
InsertTokenSupplyError = "Error inserting token_supply for block number %d: %s"
MissingBlockError = "Error finding missing token_supply records starting at block %d: %s"
)
func (tsp *TokenSupplyRepository) Create(supply TokenSupply) error {
var blockId int
err := tsp.DB.Get(&blockId, `SELECT id FROM blocks WHERE number = $1 AND eth_node_id = $2`, supply.BlockNumber, tsp.NodeID)
if err != nil {
return newRepositoryError(err, GetBlockError, supply.BlockNumber)
}
_, err = tsp.DB.Exec(
`INSERT INTO token_supply (supply, token_address, block_id)
VALUES($1, $2, $3)`,
supply.Value, supply.TokenAddress, blockId)
if err != nil {
return newRepositoryError(err, InsertTokenSupplyError, supply.BlockNumber)
}
return nil
}
func (tsp *TokenSupplyRepository) MissingBlocks(startingBlock int64, highestBlock int64) ([]int64, error) {
blockNumbers := make([]int64, 0)
err := tsp.DB.Select(
&blockNumbers,
`SELECT number FROM BLOCKS
LEFT JOIN token_supply ON blocks.id = block_id
WHERE block_id ISNULL
AND eth_node_id = $1
AND number >= $2
AND number <= $3
LIMIT 20`,
tsp.NodeID,
startingBlock,
highestBlock,
)
if err != nil {
return []int64{}, newRepositoryError(err, MissingBlockError, startingBlock)
}
return blockNumbers, err
}

207
examples/erc20_watcher/every_block/repository_test.go
View File

@ -1,207 +0,0 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package every_block_test
import (
"math/rand"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/examples/erc20_test_helpers"
"github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/test_config"
)
var _ = Describe("ERC20 Token Repository", func() {
var db *postgres.DB
var blockId int64
var blockNumber int64
var repository every_block.TokenSupplyRepository
var blockRepository repositories.BlockRepository
testAddress := "abc"
BeforeEach(func() {
node := test_config.NewTestNode()
db = test_config.NewTestDB(node)
test_config.CleanTestDB(db)
repository = every_block.TokenSupplyRepository{DB: db}
blockRepository = *repositories.NewBlockRepository(db)
blockNumber = rand.Int63()
blockId = test_config.NewTestBlock(blockNumber, blockRepository)
})
Describe("Create", func() {
It("creates a token supply record", func() {
supply := supplyModel(blockNumber, testAddress, "100")
err := repository.Create(supply)
Expect(err).NotTo(HaveOccurred())
dbResult := erc20_test_helpers.TokenSupplyDBRow{}
expectedTokenSupply := erc20_test_helpers.TokenSupplyDBRow{
Supply: int64(100),
BlockID: blockId,
TokenAddress: testAddress,
}
var count int
err = repository.DB.QueryRowx(`SELECT count(*) FROM token_supply`).Scan(&count)
Expect(err).NotTo(HaveOccurred())
Expect(count).To(Equal(1))
err = repository.DB.QueryRowx(`SELECT * FROM token_supply`).StructScan(&dbResult)
Expect(err).NotTo(HaveOccurred())
Expect(dbResult.Supply).To(Equal(expectedTokenSupply.Supply))
Expect(dbResult.BlockID).To(Equal(expectedTokenSupply.BlockID))
Expect(dbResult.TokenAddress).To(Equal(expectedTokenSupply.TokenAddress))
})
It("returns an error if fetching the block's id from the database fails", func() {
errorSupply := supplyModel(-1, "", "")
err := repository.Create(errorSupply)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("sql"))
Expect(err.Error()).To(ContainSubstring("block number -1"))
})
It("returns an error if inserting the token_supply fails", func() {
errorSupply := supplyModel(blockNumber, "", "")
err := repository.Create(errorSupply)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("pq"))
Expect(err.Error()).To(ContainSubstring("token_supply for block number"))
})
})
Describe("When there are multiple nodes", func() {
var node2DB *postgres.DB
var node2BlockRepo *repositories.BlockRepository
var node2BlockId int64
var node2TokenSupplyRepo every_block.TokenSupplyRepository
var tokenSupply every_block.TokenSupply
BeforeEach(func() {
node2DB = createDbForAnotherNode()
//create another block with the same number on node2
node2BlockRepo = repositories.NewBlockRepository(node2DB)
node2BlockId = test_config.NewTestBlock(blockNumber, *node2BlockRepo)
tokenSupply = supplyModel(blockNumber, "abc", "100")
node2TokenSupplyRepo = every_block.TokenSupplyRepository{DB: node2DB}
})
It("only creates token_supply records for the current node (node2)", func() {
err := node2TokenSupplyRepo.Create(tokenSupply)
Expect(err).NotTo(HaveOccurred())
var tokenSupplies []erc20_test_helpers.TokenSupplyDBRow
err = node2TokenSupplyRepo.DB.Select(&tokenSupplies, `SELECT * FROM token_supply`)
Expect(err).NotTo(HaveOccurred())
Expect(len(tokenSupplies)).To(Equal(1))
Expect(tokenSupplies[0].BlockID).To(Equal(node2BlockId))
})
It("only includes missing block numbers for the current node", func() {
//create token_supply on original node
err := repository.Create(tokenSupply)
Expect(err).NotTo(HaveOccurred())
originalNodeMissingBlocks, err := repository.MissingBlocks(blockNumber, blockNumber)
Expect(err).NotTo(HaveOccurred())
Expect(len(originalNodeMissingBlocks)).To(Equal(0))
node2MissingBlocks, err := node2TokenSupplyRepo.MissingBlocks(blockNumber, blockNumber)
Expect(err).NotTo(HaveOccurred())
Expect(len(node2MissingBlocks)).To(Equal(1))
})
})
Describe("MissingBlocks", func() {
It("returns the block numbers for which an associated TokenSupply record hasn't been created", func() {
createTokenSupplyFor(repository, blockNumber)
newBlockNumber := blockNumber + 1
test_config.NewTestBlock(newBlockNumber, blockRepository)
blocks, err := repository.MissingBlocks(blockNumber, newBlockNumber)
Expect(blocks).To(ConsistOf(newBlockNumber))
Expect(err).NotTo(HaveOccurred())
})
It("only returns blocks within the given range", func() {
newBlockNumber := blockNumber + 1
test_config.NewTestBlock(newBlockNumber, blockRepository)
blocks, err := repository.MissingBlocks(blockNumber, blockNumber)
Expect(blocks).NotTo(ConsistOf(newBlockNumber))
Expect(err).NotTo(HaveOccurred())
})
It("does not return numbers that already have an associated TokenSupply record", func() {
createTokenSupplyFor(repository, blockNumber)
blocks, err := repository.MissingBlocks(blockNumber, blockNumber)
Expect(blocks).To(BeEmpty())
Expect(err).NotTo(HaveOccurred())
})
})
It("deletes the token supply record when the associated block is deleted", func() {
err := repository.Create(every_block.TokenSupply{BlockNumber: blockNumber, Value: "0"})
Expect(err).NotTo(HaveOccurred())
var count int
err = repository.DB.QueryRowx(`SELECT count(*) FROM token_supply`).Scan(&count)
Expect(err).NotTo(HaveOccurred())
Expect(count).To(Equal(1))
_, err = db.Query(`DELETE FROM blocks`)
Expect(err).NotTo(HaveOccurred())
err = repository.DB.QueryRowx(`SELECT count(*) FROM token_supply`).Scan(&count)
Expect(err).NotTo(HaveOccurred())
Expect(count).To(Equal(0))
})
})
func supplyModel(blockNumber int64, tokenAddress string, supplyValue string) every_block.TokenSupply {
return every_block.TokenSupply{
Value: supplyValue,
TokenAddress: tokenAddress,
BlockNumber: int64(blockNumber),
}
}
func createTokenSupplyFor(repository every_block.TokenSupplyRepository, blockNumber int64) {
err := repository.Create(every_block.TokenSupply{BlockNumber: blockNumber, Value: "0"})
Expect(err).NotTo(HaveOccurred())
}
func createDbForAnotherNode() *postgres.DB {
anotherNode := core.Node{
GenesisBlock: "GENESIS",
NetworkID: 1,
ID: "testNodeId",
ClientName: "Geth/v1.7.2-stable-1db4ecdc/darwin-amd64/go1.9",
}
return test_config.NewTestDB(anotherNode)
}

118
examples/erc20_watcher/every_block/transformer.go
View File

@ -1,118 +0,0 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package every_block
import (
"fmt"
"math/big"
log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/examples/erc20_watcher"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
)
type Transformer struct {
Fetcher ERC20FetcherInterface
Repository ERC20RepositoryInterface
Config erc20_watcher.ContractConfig
}
func (t *Transformer) SetConfiguration(config erc20_watcher.ContractConfig) {
t.Config = config
}
type TokenSupplyTransformerInitializer struct {
Config erc20_watcher.ContractConfig
}
func (i TokenSupplyTransformerInitializer) NewTokenSupplyTransformer(db *postgres.DB, blockchain core.BlockChain) shared.Transformer {
fetcher := NewFetcher(blockchain)
repository := TokenSupplyRepository{DB: db}
transformer := Transformer{
Fetcher: &fetcher,
Repository: &repository,
Config: i.Config,
}
return transformer
}
const (
FetchingBlocksError = "Error fetching missing blocks starting at block number %d: %s"
FetchingSupplyError = "Error fetching supply for block %d: %s"
CreateSupplyError = "Error inserting token_supply for block %d: %s"
)
type transformerError struct {
err string
blockNumber int64
msg string
}
func (te *transformerError) Error() string {
return fmt.Sprintf(te.msg, te.blockNumber, te.err)
}
func newTransformerError(err error, blockNumber int64, msg string) error {
e := transformerError{err.Error(), blockNumber, msg}
log.Println(e.Error())
return &e
}
func (t Transformer) Execute() error {
var upperBoundBlock int64
blockchain := t.Fetcher.GetBlockChain()
lastBlock := blockchain.LastBlock().Int64()
if t.Config.LastBlock == -1 {
upperBoundBlock = lastBlock
} else {
upperBoundBlock = t.Config.LastBlock
}
blocks, err := t.Repository.MissingBlocks(t.Config.FirstBlock, upperBoundBlock)
if err != nil {
return newTransformerError(err, t.Config.FirstBlock, FetchingBlocksError)
}
log.Printf("Fetching totalSupply for %d blocks", len(blocks))
for _, blockNumber := range blocks {
totalSupply, err := t.Fetcher.FetchSupplyOf(t.Config.Abi, t.Config.Address, blockNumber)
if err != nil {
return newTransformerError(err, blockNumber, FetchingSupplyError)
}
model := createTokenSupplyModel(totalSupply, t.Config.Address, blockNumber)
err = t.Repository.Create(model)
if err != nil {
return newTransformerError(err, blockNumber, CreateSupplyError)
}
}
return nil
}
func createTokenSupplyModel(totalSupply big.Int, address string, blockNumber int64) TokenSupply {
return TokenSupply{
Value: totalSupply.String(),
TokenAddress: address,
BlockNumber: blockNumber,
}
}

188
examples/erc20_watcher/every_block/transformer_test.go
View File

@ -1,188 +0,0 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package every_block_test
import (
"math/big"
"math/rand"
"strconv"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/examples/constants"
"github.com/vulcanize/vulcanizedb/examples/erc20_test_helpers/mocks"
"github.com/vulcanize/vulcanizedb/examples/erc20_watcher"
"github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
)
var testContractConfig = erc20_watcher.ContractConfig{
Address: constants.DaiContractAddress,
Abi: constants.DaiAbiString,
FirstBlock: int64(4752008),
LastBlock: int64(5750050),
Name: "Dai",
}
var config = testContractConfig
var _ = Describe("Everyblock transformer", func() {
var fetcher mocks.Fetcher
var repository mocks.TotalSupplyRepository
var transformer every_block.Transformer
var blockChain *fakes.MockBlockChain
var initialSupply = "27647235749155415536952630"
var initialSupplyPlusOne = "27647235749155415536952631"
var initialSupplyPlusTwo = "27647235749155415536952632"
var initialSupplyPlusThree = "27647235749155415536952633"
var defaultLastBlock = big.Int{}
BeforeEach(func() {
blockChain = fakes.NewMockBlockChain()
blockChain.SetLastBlock(&defaultLastBlock)
fetcher = mocks.Fetcher{BlockChain: blockChain}
fetcher.SetSupply(initialSupply)
repository = mocks.TotalSupplyRepository{}
repository.SetMissingBlocks([]int64{config.FirstBlock})
//setting the mock repository to return the first block as the missing blocks
transformer = every_block.Transformer{
Fetcher: &fetcher,
Repository: &repository,
}
transformer.SetConfiguration(config)
})
It("fetches and persists the total supply of a token for a single block", func() {
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(len(fetcher.FetchedBlocks)).To(Equal(1))
Expect(fetcher.FetchedBlocks).To(ConsistOf(config.FirstBlock))
Expect(fetcher.Abi).To(Equal(config.Abi))
Expect(fetcher.ContractAddress).To(Equal(config.Address))
Expect(repository.StartingBlock).To(Equal(config.FirstBlock))
Expect(repository.EndingBlock).To(Equal(config.LastBlock))
Expect(len(repository.TotalSuppliesCreated)).To(Equal(1))
expectedSupply := big.Int{}
expectedSupply.SetString(initialSupply, 10)
expectedSupply.Add(&expectedSupply, big.NewInt(1))
Expect(repository.TotalSuppliesCreated[0].Value).To(Equal(expectedSupply.String()))
})
It("retrieves the total supply for every missing block", func() {
missingBlocks := []int64{
config.FirstBlock,
config.FirstBlock + 1,
config.FirstBlock + 2,
}
repository.SetMissingBlocks(missingBlocks)
transformer.Execute()
Expect(len(fetcher.FetchedBlocks)).To(Equal(3))
Expect(fetcher.FetchedBlocks).To(ConsistOf(config.FirstBlock, config.FirstBlock+1, config.FirstBlock+2))
Expect(fetcher.Abi).To(Equal(config.Abi))
Expect(fetcher.ContractAddress).To(Equal(config.Address))
Expect(len(repository.TotalSuppliesCreated)).To(Equal(3))
Expect(repository.TotalSuppliesCreated[0].Value).To(Equal(initialSupplyPlusOne))
Expect(repository.TotalSuppliesCreated[1].Value).To(Equal(initialSupplyPlusTwo))
Expect(repository.TotalSuppliesCreated[2].Value).To(Equal(initialSupplyPlusThree))
})
It("uses the set contract configuration", func() {
repository.SetMissingBlocks([]int64{testContractConfig.FirstBlock})
transformer.SetConfiguration(testContractConfig)
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(fetcher.FetchedBlocks).To(ConsistOf(testContractConfig.FirstBlock))
Expect(fetcher.Abi).To(Equal(testContractConfig.Abi))
Expect(fetcher.ContractAddress).To(Equal(testContractConfig.Address))
Expect(repository.StartingBlock).To(Equal(testContractConfig.FirstBlock))
Expect(repository.EndingBlock).To(Equal(testContractConfig.LastBlock))
Expect(len(repository.TotalSuppliesCreated)).To(Equal(1))
expectedTokenSupply := every_block.TokenSupply{
Value: initialSupplyPlusOne,
TokenAddress: testContractConfig.Address,
BlockNumber: testContractConfig.FirstBlock,
}
Expect(repository.TotalSuppliesCreated[0]).To(Equal(expectedTokenSupply))
})
It("uses the most recent block if the Config.LastBlock is -1", func() {
testContractConfig.LastBlock = -1
transformer.SetConfiguration(testContractConfig)
randomBlockNumber := rand.Int63()
numberToString := strconv.FormatInt(randomBlockNumber, 10)
mostRecentBlock := big.Int{}
mostRecentBlock.SetString(numberToString, 10)
blockChain.SetLastBlock(&mostRecentBlock)
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(repository.EndingBlock).To(Equal(randomBlockNumber))
})
It("returns an error if the call to get missing blocks fails", func() {
failureRepository := mocks.FailureRepository{}
failureRepository.SetMissingBlocksFail(true)
transformer = every_block.Transformer{
Fetcher: &fetcher,
Repository: &failureRepository,
}
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring(fakes.FakeError.Error()))
Expect(err.Error()).To(ContainSubstring("fetching missing blocks"))
})
It("returns an error if the call to the blockChain fails", func() {
failureBlockchain := fakes.NewMockBlockChain()
failureBlockchain.SetLastBlock(&defaultLastBlock)
failureBlockchain.SetFetchContractDataErr(fakes.FakeError)
fetcher := every_block.NewFetcher(failureBlockchain)
transformer = every_block.Transformer{
Fetcher: &fetcher,
Repository: &repository,
}
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring(fakes.FakeError.Error()))
Expect(err.Error()).To(ContainSubstring("supply"))
})
It("returns an error if the call to save the token_supply fails", func() {
failureRepository := mocks.FailureRepository{}
failureRepository.SetMissingBlocks([]int64{config.FirstBlock})
failureRepository.SetCreateFail(true)
transformer = every_block.Transformer{
Fetcher: &fetcher,
Repository: &failureRepository,
}
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring(fakes.FakeError.Error()))
Expect(err.Error()).To(ContainSubstring("supply"))
})
})

28
examples/erc20_watcher/every_block/transformers.go
View File

@ -1,28 +0,0 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package every_block
import (
"github.com/vulcanize/vulcanizedb/examples/erc20_watcher"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
)
func TransformerInitializers() []shared.TransformerInitializer {
config := erc20_watcher.DaiConfig
initializer := TokenSupplyTransformerInitializer{config}
return []shared.TransformerInitializer{
initializer.NewTokenSupplyTransformer,
}
}

81
libraries/shared/watcher.go
View File

@ -1,6 +1,8 @@
package shared
import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/log"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
@ -8,21 +10,82 @@ import (
type Watcher struct {
Transformers []shared.Transformer
DB postgres.DB
Blockchain core.BlockChain
DB *postgres.DB
Fetcher shared.LogFetcher
Chunker shared.Chunker
Addresses []common.Address
Topics []common.Hash
}
func (watcher *Watcher) AddTransformers(us []shared.TransformerInitializer) {
for _, transformerInitializer := range us {
transformer := transformerInitializer(&watcher.DB, watcher.Blockchain)
watcher.Transformers = append(watcher.Transformers, transformer)
func NewWatcher(db *postgres.DB, bc core.BlockChain) Watcher {
chunker := shared.NewLogChunker()
fetcher := shared.NewFetcher(bc)
return Watcher{
DB: db,
Fetcher: fetcher,
Chunker: chunker,
}
}
// Adds transformers to the watcher and updates the chunker, so that it will consider the new transformers.
func (watcher *Watcher) AddTransformers(initializers []shared.TransformerInitializer) {
var contractAddresses []common.Address
var topic0s []common.Hash
var configs []shared.TransformerConfig
for _, initializer := range initializers {
transformer := initializer(watcher.DB)
watcher.Transformers = append(watcher.Transformers, transformer)
config := transformer.GetConfig()
configs = append(configs, config)
addresses := shared.HexStringsToAddresses(config.ContractAddresses)
contractAddresses = append(contractAddresses, addresses...)
topic0s = append(topic0s, common.HexToHash(config.Topic))
}
watcher.Addresses = append(watcher.Addresses, contractAddresses...)
watcher.Topics = append(watcher.Topics, topic0s...)
watcher.Chunker.AddConfigs(configs)
}
func (watcher *Watcher) Execute() error {
var err error
for _, transformer := range watcher.Transformers {
err = transformer.Execute()
checkedColumnNames, err := shared.GetCheckedColumnNames(watcher.DB)
if err != nil {
return err
}
notCheckedSQL := shared.CreateNotCheckedSQL(checkedColumnNames)
// TODO Handle start and end numbers in transformers
missingHeaders, err := shared.MissingHeaders(0, -1, watcher.DB, notCheckedSQL)
if err != nil {
log.Error("Fetching of missing headers failed in watcher!")
return err
}
for _, header := range missingHeaders {
// TODO Extend FetchLogs for doing several blocks at a time
logs, err := watcher.Fetcher.FetchLogs(watcher.Addresses, watcher.Topics, header)
if err != nil {
// TODO Handle fetch error in watcher
log.Error("Error while fetching logs for header %v in watcher", header.Id)
return err
}
chunkedLogs := watcher.Chunker.ChunkLogs(logs)
// Can't quit early and mark as checked if there are no logs. If we are running continuousLogSync,
// not all logs we're interested in might have been fetched.
for _, transformer := range watcher.Transformers {
transformerName := transformer.GetConfig().TransformerName
logChunk := chunkedLogs[transformerName]
err = transformer.Execute(logChunk, header)
if err != nil {
log.Error("%v transformer failed to execute in watcher: %v", transformerName, err)
return err
}
}
}
return err
}

173
libraries/shared/watcher_test.go
View File

@ -2,69 +2,162 @@ package shared_test
import (
"errors"
"github.com/ethereum/go-ethereum"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/libraries/shared"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
shared2 "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks"
"github.com/vulcanize/vulcanizedb/test_config"
)
type MockTransformer struct {
executeWasCalled bool
executeError error
}
func (mh *MockTransformer) Execute() error {
if mh.executeError != nil {
return mh.executeError
}
mh.executeWasCalled = true
return nil
}
func fakeTransformerInitializer(db *postgres.DB, blockchain core.BlockChain) shared2.Transformer {
return &MockTransformer{}
}
var _ = Describe("Watcher", func() {
It("Adds transformers", func() {
watcher := shared.Watcher{}
It("initialises correctly", func() {
db := test_config.NewTestDB(core.Node{ID: "testNode"})
bc := fakes.NewMockBlockChain()
watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformerInitializer})
watcher := shared.NewWatcher(db, bc)
Expect(watcher.DB).To(Equal(db))
Expect(watcher.Fetcher).NotTo(BeNil())
Expect(watcher.Chunker).NotTo(BeNil())
})
It("adds transformers", func() {
watcher := shared.NewWatcher(nil, nil)
fakeTransformer := &mocks.MockTransformer{}
fakeTransformer.SetTransformerConfig(mocks.FakeTransformerConfig)
watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformer.FakeTransformerInitializer})
Expect(len(watcher.Transformers)).To(Equal(1))
Expect(watcher.Transformers).To(ConsistOf(&MockTransformer{}))
Expect(watcher.Transformers).To(ConsistOf(fakeTransformer))
Expect(watcher.Topics).To(Equal([]common.Hash{common.HexToHash("FakeTopic")}))
Expect(watcher.Addresses).To(Equal([]common.Address{common.HexToAddress("FakeAddress")}))
})
It("Adds transformers from multiple sources", func() {
watcher := shared.Watcher{}
It("adds transformers from multiple sources", func() {
watcher := shared.NewWatcher(nil, nil)
fakeTransformer1 := &mocks.MockTransformer{}
fakeTransformer1.SetTransformerConfig(mocks.FakeTransformerConfig)
watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformerInitializer})
watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformerInitializer})
fakeTransformer2 := &mocks.MockTransformer{}
fakeTransformer2.SetTransformerConfig(mocks.FakeTransformerConfig)
watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformer1.FakeTransformerInitializer})
watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformer2.FakeTransformerInitializer})
Expect(len(watcher.Transformers)).To(Equal(2))
Expect(watcher.Topics).To(Equal([]common.Hash{common.HexToHash("FakeTopic"),
common.HexToHash("FakeTopic")}))
Expect(watcher.Addresses).To(Equal([]common.Address{common.HexToAddress("FakeAddress"),
common.HexToAddress("FakeAddress")}))
})
It("Executes each transformer", func() {
watcher := shared.Watcher{}
fakeTransformer := &MockTransformer{}
watcher.Transformers = []shared2.Transformer{fakeTransformer}
Describe("with missing headers", func() {
var (
db *postgres.DB
watcher shared.Watcher
mockBlockChain fakes.MockBlockChain
fakeTransformer *mocks.MockTransformer
headerRepository repositories.HeaderRepository
repository mocks.MockWatcherRepository
)
watcher.Execute()
BeforeEach(func() {
db = test_config.NewTestDB(test_config.NewTestNode())
test_config.CleanTestDB(db)
mockBlockChain = fakes.MockBlockChain{}
headerRepository = repositories.NewHeaderRepository(db)
_, err := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
Expect(err).NotTo(HaveOccurred())
Expect(fakeTransformer.executeWasCalled).To(BeTrue())
})
repository = mocks.MockWatcherRepository{}
watcher = shared.NewWatcher(db, &mockBlockChain)
})
It("Returns an error if transformer returns an error", func() {
watcher := shared.Watcher{}
fakeTransformer := &MockTransformer{executeError: errors.New("Something bad happened")}
watcher.Transformers = []shared2.Transformer{fakeTransformer}
It("executes each transformer", func() {
fakeTransformer = &mocks.MockTransformer{}
watcher.Transformers = []shared2.Transformer{fakeTransformer}
repository.SetMissingHeaders([]core.Header{fakes.FakeHeader})
err := watcher.Execute()
err := watcher.Execute()
Expect(err).To(HaveOccurred())
Expect(fakeTransformer.executeWasCalled).To(BeFalse())
Expect(err).NotTo(HaveOccurred())
Expect(fakeTransformer.ExecuteWasCalled).To(BeTrue())
})
It("returns an error if transformer returns an error", func() {
fakeTransformer = &mocks.MockTransformer{ExecuteError: errors.New("Something bad happened")}
watcher.Transformers = []shared2.Transformer{fakeTransformer}
repository.SetMissingHeaders([]core.Header{fakes.FakeHeader})
err := watcher.Execute()
Expect(err).To(HaveOccurred())
Expect(fakeTransformer.ExecuteWasCalled).To(BeFalse())
})
It("passes only relevant logs to each transformer", func() {
transformerA := &mocks.MockTransformer{}
transformerB := &mocks.MockTransformer{}
configA := shared2.TransformerConfig{TransformerName: "transformerA",
ContractAddresses: []string{"0x000000000000000000000000000000000000000A"},
Topic: "0xA"}
configB := shared2.TransformerConfig{TransformerName: "transformerB",
ContractAddresses: []string{"0x000000000000000000000000000000000000000b"},
Topic: "0xB"}
transformerA.SetTransformerConfig(configA)
transformerB.SetTransformerConfig(configB)
logA := types.Log{Address: common.HexToAddress("0xA"),
Topics: []common.Hash{common.HexToHash("0xA")}}
logB := types.Log{Address: common.HexToAddress("0xB"),
Topics: []common.Hash{common.HexToHash("0xB")}}
mockBlockChain.SetGetEthLogsWithCustomQueryReturnLogs([]types.Log{logA, logB})
repository.SetMissingHeaders([]core.Header{fakes.FakeHeader})
watcher = shared.NewWatcher(db, &mockBlockChain)
watcher.AddTransformers([]shared2.TransformerInitializer{
transformerA.FakeTransformerInitializer, transformerB.FakeTransformerInitializer})
err := watcher.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(transformerA.PassedLogs).To(Equal([]types.Log{logA}))
Expect(transformerB.PassedLogs).To(Equal([]types.Log{logB}))
})
Describe("uses the LogFetcher correctly:", func() {
BeforeEach(func() {
repository.SetMissingHeaders([]core.Header{fakes.FakeHeader})
})
It("fetches logs", func() {
err := watcher.Execute()
Expect(err).NotTo(HaveOccurred())
fakeHash := common.HexToHash(fakes.FakeHeader.Hash)
mockBlockChain.AssertGetEthLogsWithCustomQueryCalledWith(ethereum.FilterQuery{
BlockHash: &fakeHash,
Addresses: nil,
Topics: [][]common.Hash{nil},
})
})
It("propagates log fetcher errors", func() {
fetcherError := errors.New("FetcherError")
mockBlockChain.SetGetEthLogsWithCustomQueryErr(fetcherError)
err := watcher.Execute()
Expect(err).To(MatchError(fetcherError))
})
})
})
})

1
pkg/datastore/postgres/repositories/header_repository.go
View File

@ -3,7 +3,6 @@ package repositories
import (
"database/sql"
"errors"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)

19
pkg/transformers/DOCUMENTATION.md
View File

@ -25,7 +25,7 @@ The transformer process for each of these different log types is the same, excep
## Creating a Transformer
1. Pull an example event (from kovan / ganache etc.)
1. Add event sig to [`constants.go`](./shared/constants.go)
1. Add event & method sig, contract address, `checked_headers` column name, and label to relevant files in [`constants`](./shared/constants)
1. Write a test for the event sig in [`event_signature_generator_test.go`](./shared/event_signature_generator_test.go)
1. Create DB table (using [`create_migration`](../../scripts/create_migration))
1. Create columns in `checked_headers` in the _same_ migration
@ -36,23 +36,23 @@ The transformer process for each of these different log types is the same, excep
1. Write repository + repository tests
1. Create converter + repository mocks
1. Create an config object [`shared.TransformerConfig`](./shared/transformer.go) in `config.go`
1. Create transformer + transformer tests
1. Wire up transformer in [`transformers.go`](./transformers.go)
1. Wire up transformer in [`transformers.go`](./transformers.go), remembering to add it to `TransformerInitializers()`
1. Wire up transformer in [`continuousLogSync.go`](../../cmd/continuousLogSync.go)
1. Manually trigger an event and check that it gets persisted to postgres
1. Create an integration test for the shiny new transformer in [`integration_tests`](./integration_tests)
**Fetching Logs**
1. Generate an example raw log event, by either:
- Pulling the log directly from the Kovan deployment ([constants.go](https://github.com/8thlight/maker-vulcanizedb/blob/master/pkg/transformers/shared/constants.go)).
- Pulling the log directly from the Kovan deployment ([address.go](https://github.com/8thlight/maker-vulcanizedb/blob/master/pkg/transformers/shared/constants/address.go)).
- Deploying the contract to a local chain and emiting the event manually.
1. Fetch the logs from the chain based on the example event's topic zero:
- The topic zero is based on the keccak-256 hash of the log event's method signature. These are located in [`pkg/transformers/shared/constants.go`](./shared/constants.go).
- Most transformers use `shared.LogFetcher` to fetch all logs that match the given topic zero for that log event.
- Since there are multiple price feed contract address that all use the same `LogValue` event, we have a special implementation of a fetcher specifically for price feeds that can query using all of the contract addresses at once, thus only needing to make one call to the blockchain.
- The topic zero is based on the keccak-256 hash of the log event's method signature. These are located in [`pkg/transformers/shared/constants/signature.go`](./shared/constants/signature.go).
- Fetching is done in batch from the [`watcher`](../../libraries/shared/watcher.go).
- The logs are then chunked up by the [`chunker`](./shared/log_chunker.go) before being delegated to each transformer.
**Coverting logs**
@ -105,8 +105,7 @@ The transformer process for each of these different log types is the same, excep
- The `checked_headers` table allows us to keep track of which headers have been checked for a given log type.
- To create a new migration file: `./scripts/create_migration create_flop_kick`
- See `db/migrations/1536942529_create_flop_kick.up.sql`.
- The specific log event tables are all created in the `maker`
schema.
- The specific log event tables are all created in the `maker` schema.
- There is a one-many association between `headers` and the log
event tables. This is so that if a header is removed due to a reorg, the associated log event records are also removed.
- To run the migrations: `make migrate HOST=local_host PORT=5432 NAME=vulcanize_private`
@ -120,7 +119,7 @@ The transformer process for each of these different log types is the same, excep
**Wire each component up in the transformer**
- We use a TransformerInitializer struct for each transformer so that we can inject ethRPC and postgresDB connections as well as configuration data (including the contract address, block range, etc.) into the transformer. The TransformerInitializer interface is defined in `pkg/transformers/shared/transformer.go`.
- We use a [`TransformerInitializer`](./shared/transformer.go) struct for each transformer so that we can inject ethRPC and postgresDB connections as well as configuration data (including the contract address, block range, etc.) into the transformer.
- See any of `pkg/transformers/flop_kick/transformer.go`
- All of the transformers are then initialized in `pkg/transformers/transformers.go` with their configuration.
- The transformers can be executed by using the `continuousLogSync` command, which can be configured to run specific transformers or all transformers.

5
pkg/transformers/bite/repository.go
View File

@ -16,7 +16,6 @@ package bite
import (
"fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -65,7 +64,3 @@ func (repository BiteRepository) Create(headerID int64, models []interface{}) er
func (repository BiteRepository) MarkHeaderChecked(headerID int64) error {
return shared.MarkHeaderChecked(headerID, repository.db, constants.BiteChecked)
}
func (repository BiteRepository) MissingHeaders(startingBlockNumber int64, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.BiteChecked)
}

10
pkg/transformers/bite/repository_test.go
View File

@ -86,14 +86,4 @@ var _ = Describe("Bite repository", func() {
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
})
Describe("MissingHeaders", func() {
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
Repository: &biteRepository,
RepositoryTwo: &bite.BiteRepository{},
}
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
})
})

5
pkg/transformers/cat_file/chop_lump/repository.go
View File

@ -16,7 +16,6 @@ package chop_lump
import (
"fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -62,10 +61,6 @@ func (repository CatFileChopLumpRepository) MarkHeaderChecked(headerID int64) er
return shared.MarkHeaderChecked(headerID, repository.db, constants.CatFileChopLumpChecked)
}
func (repository CatFileChopLumpRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.CatFileChopLumpChecked)
}
func (repository *CatFileChopLumpRepository) SetDB(db *postgres.DB) {
repository.db = db
}

9
pkg/transformers/cat_file/chop_lump/repository_test.go
View File

@ -98,13 +98,4 @@ var _ = Describe("Cat file chop lump repository", func() {
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
})
Describe("MissingHeaders", func() {
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
Repository: &catFileRepository,
RepositoryTwo: &chop_lump.CatFileChopLumpRepository{},
}
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
})
})

5
pkg/transformers/cat_file/flip/repository.go
View File

@ -16,7 +16,6 @@ package flip
import (
"fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -61,10 +60,6 @@ func (repository CatFileFlipRepository) MarkHeaderChecked(headerID int64) error
return shared.MarkHeaderChecked(headerID, repository.db, constants.CatFileFlipChecked)
}
func (repository CatFileFlipRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.CatFileFlipChecked)
}
func (repository *CatFileFlipRepository) SetDB(db *postgres.DB) {
repository.db = db
}

8
pkg/transformers/cat_file/flip/repository_test.go
View File

@ -83,12 +83,4 @@ var _ = Describe("Cat file flip repository", func() {
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
})
Describe("MissingHeaders", func() {
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
Repository: &catFileFlipRepository,
RepositoryTwo: &flip.CatFileFlipRepository{},
}
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
})
})

5
pkg/transformers/cat_file/pit_vow/repository.go
View File

@ -16,7 +16,6 @@ package pit_vow
import (
"fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -61,10 +60,6 @@ func (repository CatFilePitVowRepository) MarkHeaderChecked(headerID int64) erro
return shared.MarkHeaderChecked(headerID, repository.db, constants.CatFilePitVowChecked)
}
func (repository CatFilePitVowRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.CatFilePitVowChecked)
}
func (repository *CatFilePitVowRepository) SetDB(db *postgres.DB) {
repository.db = db
}

9
pkg/transformers/cat_file/pit_vow/repository_test.go
View File

@ -81,13 +81,4 @@ var _ = Describe("Cat file pit vow repository", func() {
}
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
})
Describe("MissingHeaders", func() {
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
Repository: &catFilePitVowRepository,
RepositoryTwo: &pit_vow.CatFilePitVowRepository{},
}
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
})
})

5
pkg/transformers/deal/repository.go
View File

@ -16,7 +16,6 @@ package deal
import (
"fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -62,10 +61,6 @@ func (repository DealRepository) MarkHeaderChecked(headerID int64) error {
return shared.MarkHeaderChecked(headerID, repository.db, constants.DealChecked)
}
func (repository DealRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DealChecked)
}
func (repository *DealRepository) SetDB(db *postgres.DB) {
repository.db = db
}

9
pkg/transformers/deal/repository_test.go
View File

@ -84,13 +84,4 @@ var _ = Describe("Deal Repository", func() {
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
})
Describe("MissingHeaders", func() {
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
Repository: &dealRepository,
RepositoryTwo: &deal.DealRepository{},
}
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
})
})

5
pkg/transformers/dent/repository.go
View File

@ -16,7 +16,6 @@ package dent
import (
"fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -67,10 +66,6 @@ func (repository DentRepository) MarkHeaderChecked(headerId int64) error {
return shared.MarkHeaderChecked(headerId, repository.db, constants.DentChecked)
}
func (repository DentRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DentChecked)
}
func (repository *DentRepository) SetDB(db *postgres.DB) {
repository.db = db
}

9
pkg/transformers/dent/repository_test.go
View File

@ -92,13 +92,4 @@ var _ = Describe("Dent Repository", func() {
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
})
Describe("MissingHeaders", func() {
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
Repository: &dentRepository,
RepositoryTwo: &dent.DentRepository{},
}
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
})
})

5
pkg/transformers/drip_drip/repository.go
View File

@ -16,7 +16,6 @@ package drip_drip
import (
"fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -61,10 +60,6 @@ func (repository DripDripRepository) MarkHeaderChecked(headerID int64) error {
return shared.MarkHeaderChecked(headerID, repository.db, constants.DripDripChecked)
}
func (repository DripDripRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DripDripChecked)
}
func (repository *DripDripRepository) SetDB(db *postgres.DB) {
repository.db = db
}

9
pkg/transformers/drip_drip/repository_test.go
View File

@ -80,13 +80,4 @@ var _ = Describe("Drip drip repository", func() {
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
})
Describe("MissingHeaders", func() {
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
Repository: &dripDripRepository,
RepositoryTwo: &drip_drip.DripDripRepository{},
}
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
})
})

5
pkg/transformers/drip_file/ilk/repository.go
View File

@ -16,7 +16,6 @@ package ilk
import (
"fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -64,10 +63,6 @@ func (repository DripFileIlkRepository) MarkHeaderChecked(headerID int64) error
return shared.MarkHeaderChecked(headerID, repository.db, constants.DripFileIlkChecked)
}
func (repository DripFileIlkRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DripFileIlkChecked)
}
func (repository *DripFileIlkRepository) SetDB(db *postgres.DB) {
repository.db = db
}

9
pkg/transformers/drip_file/ilk/repository_test.go
View File

@ -82,13 +82,4 @@ var _ = Describe("Drip file ilk repository", func() {
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
})
Describe("MissingHeaders", func() {
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
Repository: &dripFileIlkRepository,
RepositoryTwo: &ilk.DripFileIlkRepository{},
}
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
})
})

5
pkg/transformers/drip_file/repo/repository.go
View File

@ -16,7 +16,6 @@ package repo
import (
"fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -64,10 +63,6 @@ func (repository DripFileRepoRepository) MarkHeaderChecked(headerID int64) error
return shared.MarkHeaderChecked(headerID, repository.db, constants.DripFileRepoChecked)
}
func (repository DripFileRepoRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DripFileRepoChecked)
}
func (repository *DripFileRepoRepository) SetDB(db *postgres.DB) {
repository.db = db
}

9
pkg/transformers/drip_file/repo/repository_test.go
View File

@ -82,13 +82,4 @@ var _ = Describe("Drip file repo repository", func() {
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
})
Describe("MissingHeaders", func() {
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
Repository: &dripFileRepoRepository,
RepositoryTwo: &repo.DripFileRepoRepository{},
}
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
})
})

5
pkg/transformers/drip_file/vow/repository.go
View File

@ -16,7 +16,6 @@ package vow
import (
"fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -63,10 +62,6 @@ func (repository DripFileVowRepository) MarkHeaderChecked(headerID int64) error
return shared.MarkHeaderChecked(headerID, repository.db, constants.DripFileVowChecked)
}
func (repository DripFileVowRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DripFileVowChecked)
}
func (repository *DripFileVowRepository) SetDB(db *postgres.DB) {
repository.db = db
}

9
pkg/transformers/drip_file/vow/repository_test.go
View File

@ -74,15 +74,6 @@ var _ = Describe("Drip file vow repository", func() {
})
Describe("MarkHeaderChecked", func() {
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
Repository: &dripFileVowRepository,
RepositoryTwo: &vow.DripFileVowRepository{},
}
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
})
Describe("MissingHeaders", func() {
inputs := shared_behaviors.MarkedHeaderCheckedBehaviorInputs{
CheckedHeaderColumnName: constants.DripFileVowChecked,
Repository: &dripFileVowRepository,

71
pkg/transformers/factories/log_note_transformer.go
View File

@ -15,10 +15,9 @@
package factories
import (
"github.com/ethereum/go-ethereum/core/types"
log "github.com/sirupsen/logrus"
"github.com/ethereum/go-ethereum/common"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
@ -28,58 +27,44 @@ type LogNoteTransformer struct {
Config shared.TransformerConfig
Converter LogNoteConverter
Repository Repository
Fetcher shared.SettableLogFetcher
}
func (transformer LogNoteTransformer) NewLogNoteTransformer(db *postgres.DB, bc core.BlockChain) shared.Transformer {
func (transformer LogNoteTransformer) NewLogNoteTransformer(db *postgres.DB) shared.Transformer {
transformer.Repository.SetDB(db)
transformer.Fetcher.SetBC(bc)
return transformer
}
func (transformer LogNoteTransformer) Execute() error {
func (transformer LogNoteTransformer) Execute(logs []types.Log, header core.Header) error {
transformerName := transformer.Config.TransformerName
missingHeaders, err := transformer.Repository.MissingHeaders(transformer.Config.StartingBlockNumber, transformer.Config.EndingBlockNumber)
// No matching logs, mark the header as checked for this type of logs
if len(logs) < 1 {
err := transformer.Repository.MarkHeaderChecked(header.Id)
if err != nil {
log.Printf("Error marking header as checked in %v: %v", transformerName, err)
return err
}
return nil
}
models, err := transformer.Converter.ToModels(logs)
if err != nil {
log.Printf("Error fetching mising headers in %v transformer: %v", transformerName, err)
log.Printf("Error converting logs in %v: %v", transformerName, err)
return err
}
// Grab event signature from transformer config
// (Double-array structure required for go-ethereum FilterQuery)
var topic = [][]common.Hash{{common.HexToHash(transformer.Config.Topic)}}
log.Printf("Fetching %v event logs for %d headers", transformerName, len(missingHeaders))
for _, header := range missingHeaders {
// Fetch the missing logs for a given header
matchingLogs, err := transformer.Fetcher.FetchLogs(transformer.Config.ContractAddresses, topic, header)
if err != nil {
log.Printf("Error fetching matching logs in %v transformer: %v", transformerName, err)
return err
}
// No matching logs, mark the header as checked for this type of logs
if len(matchingLogs) < 1 {
err := transformer.Repository.MarkHeaderChecked(header.Id)
if err != nil {
log.Printf("Error marking header as checked in %v: %v", transformerName, err)
return err
}
// Continue with the next header; nothing to persist
continue
}
models, err := transformer.Converter.ToModels(matchingLogs)
if err != nil {
log.Printf("Error converting logs in %v: %v", transformerName, err)
return err
}
err = transformer.Repository.Create(header.Id, models)
if err != nil {
log.Printf("Error persisting %v record: %v", transformerName, err)
return err
}
err = transformer.Repository.Create(header.Id, models)
if err != nil {
log.Printf("Error persisting %v record: %v", transformerName, err)
return err
}
return nil
}
func (transformer LogNoteTransformer) GetName() string {
return transformer.Config.TransformerName
}
func (transformer LogNoteTransformer) GetConfig() shared.TransformerConfig {
return transformer.Config
}

86
pkg/transformers/factories/log_note_transformer_test.go
View File

@ -15,7 +15,7 @@
package factories_test
import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
@ -31,9 +31,7 @@ var _ = Describe("LogNoteTransformer", func() {
var (
repository mocks.MockRepository
converter mocks.MockLogNoteConverter
fetcher mocks.MockLogFetcher
headerOne core.Header
headerTwo core.Header
transformer shared.Transformer
model test_data.GenericModel
config = test_data.GenericTestConfig
@ -43,75 +41,28 @@ var _ = Describe("LogNoteTransformer", func() {
BeforeEach(func() {
repository = mocks.MockRepository{}
converter = mocks.MockLogNoteConverter{}
fetcher = mocks.MockLogFetcher{}
transformer = factories.LogNoteTransformer{
Config: config,
Converter: &converter,
Repository: &repository,
Fetcher: &fetcher,
}.NewLogNoteTransformer(nil, nil)
}.NewLogNoteTransformer(nil)
headerOne = core.Header{Id: rand.Int63(), BlockNumber: rand.Int63()}
headerTwo = core.Header{Id: rand.Int63(), BlockNumber: rand.Int63()}
})
It("sets the blockchain and database", func() {
Expect(fetcher.SetBcCalled).To(BeTrue())
It("sets the database", func() {
Expect(repository.SetDbCalled).To(BeTrue())
})
It("gets missing headers for block numbers specified in config", func() {
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(repository.PassedStartingBlockNumber).To(Equal(config.StartingBlockNumber))
Expect(repository.PassedEndingBlockNumber).To(Equal(config.EndingBlockNumber))
})
It("returns error if repository returns error for missing headers", func() {
repository.SetMissingHeadersError(fakes.FakeError)
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("fetches logs for missing headers", func() {
repository.SetMissingHeaders([]core.Header{headerOne, headerTwo})
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(fetcher.FetchedBlocks).To(Equal([]int64{headerOne.BlockNumber, headerTwo.BlockNumber}))
Expect(fetcher.FetchedContractAddresses).To(Equal([][]string{config.ContractAddresses, config.ContractAddresses}))
expectedTopic := common.HexToHash(config.Topic)
Expect(fetcher.FetchedTopics).To(Equal([][]common.Hash{{expectedTopic}}))
})
It("returns error if fetcher returns error", func() {
repository.SetMissingHeaders([]core.Header{headerOne})
fetcher.SetFetcherError(fakes.FakeError)
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("marks header checked if no logs returned", func() {
repository.SetMissingHeaders([]core.Header{headerOne})
err := transformer.Execute()
It("marks header checked if no logs are provided", func() {
err := transformer.Execute([]types.Log{}, headerOne)
Expect(err).NotTo(HaveOccurred())
repository.AssertMarkHeaderCheckedCalledWith(headerOne.Id)
})
It("doesn't attempt to convert or persist an empty collection when there are no logs", func() {
repository.SetMissingHeaders([]core.Header{headerOne, headerTwo})
err := transformer.Execute()
err := transformer.Execute([]types.Log{}, headerOne)
Expect(err).NotTo(HaveOccurred())
Expect(converter.ToModelsCalledCounter).To(Equal(0))
@ -119,30 +70,23 @@ var _ = Describe("LogNoteTransformer", func() {
})
It("does not call repository.MarkCheckedHeader when there are logs", func() {
repository.SetMissingHeaders([]core.Header{headerOne})
fetcher.SetFetchedLogs(logs)
err := transformer.Execute()
err := transformer.Execute(logs, headerOne)
Expect(err).NotTo(HaveOccurred())
repository.AssertMarkHeaderCheckedNotCalled()
})
It("returns error if marking header checked returns err", func() {
repository.SetMissingHeaders([]core.Header{headerOne})
repository.SetMarkHeaderCheckedError(fakes.FakeError)
err := transformer.Execute()
err := transformer.Execute([]types.Log{}, headerOne)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("converts matching logs to models", func() {
fetcher.SetFetchedLogs(logs)
repository.SetMissingHeaders([]core.Header{headerOne})
err := transformer.Execute()
err := transformer.Execute(logs, headerOne)
Expect(err).NotTo(HaveOccurred())
Expect(converter.PassedLogs).To(Equal(logs))
@ -150,20 +94,16 @@ var _ = Describe("LogNoteTransformer", func() {
It("returns error if converter returns error", func() {
converter.SetConverterError(fakes.FakeError)
fetcher.SetFetchedLogs(logs)
repository.SetMissingHeaders([]core.Header{headerOne})
err := transformer.Execute()
err := transformer.Execute(logs, headerOne)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("persists the model", func() {
fetcher.SetFetchedLogs(logs)
repository.SetMissingHeaders([]core.Header{headerOne})
converter.SetReturnModels([]interface{}{model})
err := transformer.Execute()
err := transformer.Execute(logs, headerOne)
Expect(err).NotTo(HaveOccurred())
Expect(repository.PassedHeaderID).To(Equal(headerOne.Id))
@ -171,11 +111,9 @@ var _ = Describe("LogNoteTransformer", func() {
})
It("returns error if repository returns error for create", func() {
fetcher.SetFetchedLogs(logs)
repository.SetMissingHeaders([]core.Header{headerOne})
repository.SetCreateError(fakes.FakeError)
err := transformer.Execute()
err := transformer.Execute(logs, headerOne)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))

2
pkg/transformers/factories/repository.go
View File

@ -15,13 +15,11 @@
package factories
import (
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
type Repository interface {
Create(headerID int64, models []interface{}) error
MarkHeaderChecked(headerID int64) error
MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
SetDB(db *postgres.DB)
}

75
pkg/transformers/factories/transformer.go
View File

@ -15,10 +15,9 @@
package factories
import (
"github.com/ethereum/go-ethereum/core/types"
log "github.com/sirupsen/logrus"
"github.com/ethereum/go-ethereum/common"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
@ -28,61 +27,51 @@ type Transformer struct {
Config shared.TransformerConfig
Converter Converter
Repository Repository
Fetcher shared.SettableLogFetcher
}
func (transformer Transformer) NewTransformer(db *postgres.DB, bc core.BlockChain) shared.Transformer {
func (transformer Transformer) NewTransformer(db *postgres.DB) shared.Transformer {
transformer.Repository.SetDB(db)
transformer.Fetcher.SetBC(bc)
return transformer
}
func (transformer Transformer) Execute() error {
func (transformer Transformer) Execute(logs []types.Log, header core.Header) error {
transformerName := transformer.Config.TransformerName
config := transformer.Config
topics := [][]common.Hash{{common.HexToHash(config.Topic)}}
missingHeaders, err := transformer.Repository.MissingHeaders(config.StartingBlockNumber, config.EndingBlockNumber)
if len(logs) < 1 {
err := transformer.Repository.MarkHeaderChecked(header.Id)
if err != nil {
log.Printf("Error marking header as checked in %v: %v", transformerName, err)
return err
}
return nil
}
entities, err := transformer.Converter.ToEntities(config.ContractAbi, logs)
if err != nil {
log.Printf("Error fetching missing headers in %v transformer: %v \n", transformerName, err)
log.Printf("Error converting logs to entities in %v: %v", transformerName, err)
return err
}
log.Printf("Fetching %v event logs for %d headers \n", transformerName, len(missingHeaders))
for _, header := range missingHeaders {
logs, err := transformer.Fetcher.FetchLogs(config.ContractAddresses, topics, header)
if err != nil {
log.Printf("Error fetching matching logs in %v transformer: %v", transformerName, err)
return err
}
if len(logs) < 1 {
err = transformer.Repository.MarkHeaderChecked(header.Id)
if err != nil {
log.Printf("Error marking header as checked in %v: %v", transformerName, err)
return err
}
continue
}
entities, err := transformer.Converter.ToEntities(config.ContractAbi, logs)
if err != nil {
log.Printf("Error converting logs to entities in %v: %v", transformerName, err)
return err
}
models, err := transformer.Converter.ToModels(entities)
if err != nil {
log.Printf("Error converting entities to models in %v: %v", transformerName, err)
return err
}
err = transformer.Repository.Create(header.Id, models)
if err != nil {
log.Printf("Error persisting %v record: %v", transformerName, err)
return err
}
models, err := transformer.Converter.ToModels(entities)
if err != nil {
log.Printf("Error converting entities to models in %v: %v", transformerName, err)
return err
}
err = transformer.Repository.Create(header.Id, models)
if err != nil {
log.Printf("Error persisting %v record: %v", transformerName, err)
return err
}
return nil
}
func (transformer Transformer) GetName() string {
return transformer.Config.TransformerName
}
func (transformer Transformer) GetConfig() shared.TransformerConfig {
return transformer.Config
}

91
pkg/transformers/factories/transformer_test.go
View File

@ -15,7 +15,7 @@
package factories_test
import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
@ -30,86 +30,39 @@ import (
var _ = Describe("Transformer", func() {
var (
repository mocks.MockRepository
fetcher mocks.MockLogFetcher
converter mocks.MockConverter
transformer shared.Transformer
headerOne core.Header
headerTwo core.Header
config = test_data.GenericTestConfig
logs = test_data.GenericTestLogs
)
BeforeEach(func() {
repository = mocks.MockRepository{}
fetcher = mocks.MockLogFetcher{}
converter = mocks.MockConverter{}
transformer = factories.Transformer{
Repository: &repository,
Fetcher: &fetcher,
Converter: &converter,
Config: config,
}.NewTransformer(nil, nil)
}.NewTransformer(nil)
headerOne = core.Header{Id: rand.Int63(), BlockNumber: rand.Int63()}
headerTwo = core.Header{Id: rand.Int63(), BlockNumber: rand.Int63()}
})
It("sets the blockchain and db", func() {
Expect(fetcher.SetBcCalled).To(BeTrue())
It("sets the db", func() {
Expect(repository.SetDbCalled).To(BeTrue())
})
It("gets missing headers for blocks in the configured range", func() {
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(repository.PassedStartingBlockNumber).To(Equal(config.StartingBlockNumber))
Expect(repository.PassedEndingBlockNumber).To(Equal(config.EndingBlockNumber))
})
It("returns an error if it fails to get missing headers", func() {
repository.SetMissingHeadersError(fakes.FakeError)
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("fetches eth logs for each missing header", func() {
repository.SetMissingHeaders([]core.Header{headerOne, headerTwo})
expectedTopics := [][]common.Hash{{common.HexToHash(config.Topic)}}
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(fetcher.FetchedBlocks).To(Equal([]int64{headerOne.BlockNumber, headerTwo.BlockNumber}))
Expect(fetcher.FetchedTopics).To(Equal(expectedTopics))
Expect(fetcher.FetchedContractAddresses).To(Equal([][]string{config.ContractAddresses, config.ContractAddresses}))
})
It("returns an error if fetching logs fails", func() {
repository.SetMissingHeaders([]core.Header{headerOne})
fetcher.SetFetcherError(fakes.FakeError)
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("marks header checked if no logs returned", func() {
headerID := int64(123)
repository.SetMissingHeaders([]core.Header{{Id: headerID}})
err := transformer.Execute()
err := transformer.Execute([]types.Log{}, headerOne)
Expect(err).NotTo(HaveOccurred())
repository.AssertMarkHeaderCheckedCalledWith(headerID)
repository.AssertMarkHeaderCheckedCalledWith(headerOne.Id)
})
It("doesn't attempt to convert or persist an empty collection when there are no logs", func() {
repository.SetMissingHeaders([]core.Header{headerOne, headerTwo})
err := transformer.Execute()
err := transformer.Execute([]types.Log{}, headerOne)
Expect(err).NotTo(HaveOccurred())
Expect(converter.ToEntitiesCalledCounter).To(Equal(0))
@ -118,29 +71,23 @@ var _ = Describe("Transformer", func() {
})
It("does not call repository.MarkCheckedHeader when there are logs", func() {
repository.SetMissingHeaders([]core.Header{headerOne})
fetcher.SetFetchedLogs(logs)
err := transformer.Execute()
err := transformer.Execute(logs, headerOne)
Expect(err).NotTo(HaveOccurred())
repository.AssertMarkHeaderCheckedNotCalled()
})
It("returns error if marking header checked returns err", func() {
repository.SetMissingHeaders([]core.Header{headerOne})
repository.SetMarkHeaderCheckedError(fakes.FakeError)
err := transformer.Execute()
err := transformer.Execute([]types.Log{}, headerOne)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("converts an eth log to an entity", func() {
repository.SetMissingHeaders([]core.Header{headerOne})
fetcher.SetFetchedLogs(logs)
err := transformer.Execute()
err := transformer.Execute(logs, headerOne)
Expect(err).NotTo(HaveOccurred())
Expect(converter.ContractAbi).To(Equal(config.ContractAbi))
@ -148,45 +95,37 @@ var _ = Describe("Transformer", func() {
})
It("returns an error if converter fails", func() {
repository.SetMissingHeaders([]core.Header{headerOne})
fetcher.SetFetchedLogs(logs)
converter.ToEntitiesError = fakes.FakeError
err := transformer.Execute()
err := transformer.Execute(logs, headerOne)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("converts an entity to a model", func() {
repository.SetMissingHeaders([]core.Header{headerOne})
fetcher.SetFetchedLogs(logs)
converter.EntitiesToReturn = []interface{}{test_data.GenericEntity{}}
err := transformer.Execute()
err := transformer.Execute(logs, headerOne)
Expect(err).NotTo(HaveOccurred())
Expect(converter.EntitiesToConvert[0]).To(Equal(test_data.GenericEntity{}))
})
It("returns an error if converting to models fails", func() {
repository.SetMissingHeaders([]core.Header{headerOne})
fetcher.SetFetchedLogs(logs)
converter.EntitiesToReturn = []interface{}{test_data.GenericEntity{}}
converter.ToModelsError = fakes.FakeError
err := transformer.Execute()
err := transformer.Execute(logs, headerOne)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("persists the record", func() {
repository.SetMissingHeaders([]core.Header{headerOne})
fetcher.SetFetchedLogs(logs)
converter.ModelsToReturn = []interface{}{test_data.GenericModel{}}
err := transformer.Execute()
err := transformer.Execute(logs, headerOne)
Expect(err).NotTo(HaveOccurred())
Expect(repository.PassedHeaderID).To(Equal(headerOne.Id))
@ -194,10 +133,8 @@ var _ = Describe("Transformer", func() {
})
It("returns error if persisting the record fails", func() {
repository.SetMissingHeaders([]core.Header{headerOne})
fetcher.SetFetchedLogs(logs)
repository.SetCreateError(fakes.FakeError)
err := transformer.Execute()
err := transformer.Execute(logs, headerOne)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))

5
pkg/transformers/flap_kick/repository.go
View File

@ -16,7 +16,6 @@ package flap_kick
import (
"fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -60,10 +59,6 @@ func (repository *FlapKickRepository) MarkHeaderChecked(headerID int64) error {
return shared.MarkHeaderChecked(headerID, repository.db, constants.FlapKickChecked)
}
func (repository FlapKickRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.FlapKickChecked)
}
func (repository *FlapKickRepository) SetDB(db *postgres.DB) {
repository.db = db
}

9
pkg/transformers/flap_kick/repository_test.go
View File

@ -87,13 +87,4 @@ var _ = Describe("Flap Kick Repository", func() {
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
})
Describe("MissingHeaders", func() {
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
Repository: &flapKickRepository,
RepositoryTwo: &flap_kick.FlapKickRepository{},
}
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
})
})

5
pkg/transformers/flip_kick/repository.go
View File

@ -17,7 +17,6 @@ package flip_kick
import (
"fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -60,10 +59,6 @@ func (repository FlipKickRepository) MarkHeaderChecked(headerId int64) error {
return shared.MarkHeaderChecked(headerId, repository.db, constants.FlipKickChecked)
}
func (repository FlipKickRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.FlipKickChecked)
}
func (repository *FlipKickRepository) SetDB(db *postgres.DB) {
repository.db = db
}

9
pkg/transformers/flip_kick/repository_test.go
View File

@ -87,15 +87,6 @@ var _ = Describe("FlipKick Repository", func() {
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
})
Describe("missing headers", func() {
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
Repository: &flipKickRepository,
RepositoryTwo: &flip_kick.FlipKickRepository{},
}
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
})
})
func assertDBRecordCount(db *postgres.DB, dbTable string, expectedCount int) {

5
pkg/transformers/flop_kick/repository.go
View File

@ -16,7 +16,6 @@ package flop_kick
import (
"fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -62,10 +61,6 @@ func (repository FlopKickRepository) MarkHeaderChecked(headerId int64) error {
return shared.MarkHeaderChecked(headerId, repository.db, constants.FlopKickChecked)
}
func (repository FlopKickRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.FlopKickChecked)
}
func (repository *FlopKickRepository) SetDB(db *postgres.DB) {
repository.db = db
}

9
pkg/transformers/flop_kick/repository_test.go
View File

@ -85,13 +85,4 @@ var _ = Describe("FlopRepository", func() {
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
})
Describe("MissingHeaders", func() {
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
Repository: &repository,
RepositoryTwo: &flop_kick.FlopKickRepository{},
}
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
})
})

5
pkg/transformers/frob/repository.go
View File

@ -17,7 +17,6 @@ package frob
import (
"fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -59,10 +58,6 @@ func (repository FrobRepository) MarkHeaderChecked(headerID int64) error {
return shared.MarkHeaderChecked(headerID, repository.db, constants.FrobChecked)
}
func (repository FrobRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.FrobChecked)
}
func (repository *FrobRepository) SetDB(db *postgres.DB) {
repository.db = db
}

9
pkg/transformers/frob/repository_test.go
View File

@ -86,13 +86,4 @@ var _ = Describe("Frob repository", func() {
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
})
Describe("MissingHeaders", func() {
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
Repository: &frobRepository,
RepositoryTwo: &frob.FrobRepository{},
}
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
})
})

15
pkg/transformers/integration_tests/bite.go
View File

@ -45,17 +45,24 @@ var _ = Describe("Bite Transformer", func() {
db := test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
err = persistHeader(db, blockNumber, blockChain)
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
initializer := factories.Transformer{
Config: config,
Converter: &bite.BiteConverter{},
Repository: &bite.BiteRepository{},
Fetcher: &shared.Fetcher{},
}
transformer := initializer.NewTransformer(db, blockChain)
err = transformer.Execute()
transformer := initializer.NewTransformer(db)
fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(
[]common.Address{common.HexToAddress(config.ContractAddresses[0])},
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred())
var dbResult []bite.BiteModel

47
pkg/transformers/integration_tests/cat_file.go
View File

@ -15,6 +15,7 @@
package integration_tests
import (
"github.com/ethereum/go-ethereum/common"
"github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"sort"
@ -39,6 +40,7 @@ var _ = Describe("Cat File transformer", func() {
rpcClient client.RpcClient
err error
ethClient *ethclient.Client
fetcher *shared.Fetcher
)
BeforeEach(func() {
@ -48,13 +50,15 @@ var _ = Describe("Cat File transformer", func() {
Expect(err).NotTo(HaveOccurred())
db = test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
fetcher = shared.NewFetcher(blockChain)
})
// Cat contract Kovan address: 0x2f34f22a00ee4b7a8f8bbc4eaee1658774c624e0
It("persists a chop lump event", func() {
// transaction: 0x98574bfba4d05c3875be10d2376e678d005dbebe9a4520363407508fd21f4014
chopLumpBlockNumber := int64(8762253)
err = persistHeader(db, chopLumpBlockNumber, blockChain)
header, err := persistHeader(db, chopLumpBlockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
config := chop_lump.CatFileChopLumpConfig
@ -65,10 +69,16 @@ var _ = Describe("Cat File transformer", func() {
Config: config,
Converter: &chop_lump.CatFileChopLumpConverter{},
Repository: &chop_lump.CatFileChopLumpRepository{},
Fetcher: &shared.Fetcher{},
}
transformer := initializer.NewLogNoteTransformer(db, blockChain)
err := transformer.Execute()
transformer := initializer.NewLogNoteTransformer(db)
logs, err := fetcher.FetchLogs(
[]common.Address{common.HexToAddress(config.ContractAddresses[0])},
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred())
var dbResult []chop_lump.CatFileChopLumpModel
@ -92,7 +102,7 @@ var _ = Describe("Cat File transformer", func() {
It("persists a flip event", func() {
// transaction: 0x44bc18fdb1a5a263db114e7879653304db3e19ceb4e4496f21bc0a76c5faccbe
flipBlockNumber := int64(8751794)
err = persistHeader(db, flipBlockNumber, blockChain)
header, err := persistHeader(db, flipBlockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
config := flip.CatFileFlipConfig
@ -103,10 +113,17 @@ var _ = Describe("Cat File transformer", func() {
Config: config,
Converter: &flip.CatFileFlipConverter{},
Repository: &flip.CatFileFlipRepository{},
Fetcher: &shared.Fetcher{},
}
transformer := initializer.NewLogNoteTransformer(db, blockChain)
err := transformer.Execute()
transformer := initializer.NewLogNoteTransformer(db)
logs, err := fetcher.FetchLogs(
[]common.Address{common.HexToAddress(config.ContractAddresses[0])},
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred())
var dbResult []flip.CatFileFlipModel
@ -122,7 +139,7 @@ var _ = Describe("Cat File transformer", func() {
It("persists a pit vow event", func() {
// transaction: 0x44bc18fdb1a5a263db114e7879653304db3e19ceb4e4496f21bc0a76c5faccbe
pitVowBlockNumber := int64(8751794)
err = persistHeader(db, pitVowBlockNumber, blockChain)
header, err := persistHeader(db, pitVowBlockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
config := pit_vow.CatFilePitVowConfig
@ -133,10 +150,16 @@ var _ = Describe("Cat File transformer", func() {
Config: config,
Converter: &pit_vow.CatFilePitVowConverter{},
Repository: &pit_vow.CatFilePitVowRepository{},
Fetcher: &shared.Fetcher{},
}
transformer := initializer.NewLogNoteTransformer(db, blockChain)
err := transformer.Execute()
transformer := initializer.NewLogNoteTransformer(db)
logs, err := fetcher.FetchLogs(
[]common.Address{common.HexToAddress(config.ContractAddresses[0])},
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred())
var dbResult []pit_vow.CatFilePitVowModel

63
pkg/transformers/integration_tests/deal.go
View File

@ -15,6 +15,7 @@
package integration_tests
import (
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
@ -29,8 +30,13 @@ import (
var _ = Describe("Deal transformer", func() {
var (
db *postgres.DB
blockChain core.BlockChain
db *postgres.DB
blockChain core.BlockChain
config shared.TransformerConfig
initializer factories.LogNoteTransformer
fetcher *shared.Fetcher
addresses []common.Address
topics []common.Hash
)
BeforeEach(func() {
@ -40,26 +46,35 @@ var _ = Describe("Deal transformer", func() {
Expect(err).NotTo(HaveOccurred())
db = test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
config = deal.DealConfig
initializer = factories.LogNoteTransformer{
Config: config,
Converter: &deal.DealConverter{},
Repository: &deal.DealRepository{},
}
fetcher = shared.NewFetcher(blockChain)
addresses = shared.HexStringsToAddresses(config.ContractAddresses)
topics = []common.Hash{common.HexToHash(config.Topic)}
})
It("persists a flip deal log event", func() {
// transaction: 0x05b5eabac2ace136f0f7e0efc61d7d42abe8e8938cc0f04fbf1a6ba545d59e58
flipBlockNumber := int64(8958007)
err := persistHeader(db, flipBlockNumber, blockChain)
header, err := persistHeader(db, flipBlockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
config := deal.DealConfig
config.StartingBlockNumber = flipBlockNumber
config.EndingBlockNumber = flipBlockNumber
initializer.Config.StartingBlockNumber = flipBlockNumber
initializer.Config.EndingBlockNumber = flipBlockNumber
initializer := factories.LogNoteTransformer{
Config: config,
Converter: &deal.DealConverter{},
Repository: &deal.DealRepository{},
Fetcher: &shared.Fetcher{},
}
transformer := initializer.NewLogNoteTransformer(db, blockChain)
err = transformer.Execute()
logs, err := fetcher.FetchLogs(addresses, topics, header)
Expect(err).NotTo(HaveOccurred())
transformer := initializer.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred())
var dbResult []deal.DealModel
@ -77,21 +92,17 @@ var _ = Describe("Deal transformer", func() {
It("persists a flap deal log event", func() {
flapBlockNumber := int64(9004628)
err := persistHeader(db, flapBlockNumber, blockChain)
header, err := persistHeader(db, flapBlockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
config := deal.DealConfig
config.StartingBlockNumber = flapBlockNumber
config.EndingBlockNumber = flapBlockNumber
initializer.Config.StartingBlockNumber = flapBlockNumber
initializer.Config.EndingBlockNumber = flapBlockNumber
initializer := factories.LogNoteTransformer{
Config: config,
Converter: &deal.DealConverter{},
Repository: &deal.DealRepository{},
Fetcher: &shared.Fetcher{},
}
transformer := initializer.NewLogNoteTransformer(db, blockChain)
err = transformer.Execute()
logs, err := fetcher.FetchLogs(addresses, topics, header)
Expect(err).NotTo(HaveOccurred())
transformer := initializer.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred())
var dbResult []deal.DealModel

42
pkg/transformers/integration_tests/dent.go
View File

@ -1,6 +1,7 @@
package integration_tests
import (
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -15,8 +16,14 @@ import (
var _ = Describe("Dent transformer", func() {
var (
db *postgres.DB
blockChain core.BlockChain
db *postgres.DB
blockChain core.BlockChain
fetcher *shared.Fetcher
transformer shared.Transformer
config shared.TransformerConfig
addresses []common.Address
topics []common.Hash
initializer factories.LogNoteTransformer
)
BeforeEach(func() {
@ -26,25 +33,32 @@ var _ = Describe("Dent transformer", func() {
Expect(err).NotTo(HaveOccurred())
db = test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
config = dent.DentConfig
addresses = shared.HexStringsToAddresses(config.ContractAddresses)
topics = []common.Hash{common.HexToHash(config.Topic)}
fetcher = shared.NewFetcher(blockChain)
initializer = factories.LogNoteTransformer{
Config: config,
Converter: &dent.DentConverter{},
Repository: &dent.DentRepository{},
}
})
It("persists a flop dent log event", func() {
blockNumber := int64(8955613)
err := persistHeader(db, blockNumber, blockChain)
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
config := dent.DentConfig
config.StartingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber
initializer.Config.StartingBlockNumber = blockNumber
initializer.Config.EndingBlockNumber = blockNumber
initializer := factories.LogNoteTransformer{
Config: config,
Converter: &dent.DentConverter{},
Repository: &dent.DentRepository{},
Fetcher: &shared.Fetcher{},
}
transformer := initializer.NewLogNoteTransformer(db, blockChain)
err = transformer.Execute()
logs, err := fetcher.FetchLogs(addresses, topics, header)
Expect(err).NotTo(HaveOccurred())
transformer = initializer.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred())
var dbResult []dent.DentModel

38
pkg/transformers/integration_tests/drip_drip.go
View File

@ -15,8 +15,11 @@
package integration_tests
import (
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
@ -25,32 +28,43 @@ import (
)
var _ = Describe("DripDrip Transformer", func() {
var (
db *postgres.DB
blockChain core.BlockChain
)
BeforeEach(func() {
rpcClient, ethClient, err := getClients(ipc)
Expect(err).NotTo(HaveOccurred())
blockChain, err = getBlockChain(rpcClient, ethClient)
Expect(err).NotTo(HaveOccurred())
db = test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
})
It("transforms DripDrip log events", func() {
blockNumber := int64(8934775)
config := drip_drip.DripDripConfig
config.StartingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber
rpcClient, ethClient, err := getClients(ipc)
Expect(err).NotTo(HaveOccurred())
blockChain, err := getBlockChain(rpcClient, ethClient)
Expect(err).NotTo(HaveOccurred())
db := test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
err = persistHeader(db, blockNumber, blockChain)
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
initializer := factories.LogNoteTransformer{
Config: config,
Converter: &drip_drip.DripDripConverter{},
Repository: &drip_drip.DripDripRepository{},
Fetcher: &shared.Fetcher{},
}
transformer := initializer.NewLogNoteTransformer(db, blockChain)
transformer := initializer.NewLogNoteTransformer(db)
err = transformer.Execute()
fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred())
var dbResults []drip_drip.DripDripModel

40
pkg/transformers/integration_tests/drip_file_vow.go
View File

@ -15,8 +15,11 @@
package integration_tests
import (
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
@ -25,31 +28,44 @@ import (
)
var _ = Describe("Drip File Vow LogNoteTransformer", func() {
var (
db *postgres.DB
blockChain core.BlockChain
)
BeforeEach(func() {
rpcClient, ethClient, err := getClients(ipc)
Expect(err).NotTo(HaveOccurred())
blockChain, err = getBlockChain(rpcClient, ethClient)
Expect(err).NotTo(HaveOccurred())
db = test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
})
It("transforms DripFileVow log events", func() {
blockNumber := int64(8762197)
config := vow.DripFileVowConfig
config.StartingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber
rpcClient, ethClient, err := getClients(ipc)
Expect(err).NotTo(HaveOccurred())
blockChain, err := getBlockChain(rpcClient, ethClient)
Expect(err).NotTo(HaveOccurred())
db := test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
err = persistHeader(db, blockNumber, blockChain)
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
initializer := factories.LogNoteTransformer{
Config: config,
Fetcher: &shared.Fetcher{},
Converter: &vow.DripFileVowConverter{},
Repository: &vow.DripFileVowRepository{},
}
transformer := initializer.NewLogNoteTransformer(db, blockChain)
err = transformer.Execute()
transformer := initializer.NewLogNoteTransformer(db)
fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred())
var dbResult []vow.DripFileVowModel

45
pkg/transformers/integration_tests/flap_kick.go
View File

@ -15,6 +15,10 @@
package integration_tests
import (
"github.com/ethereum/go-ethereum/common"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/test_config"
"time"
. "github.com/onsi/ginkgo"
@ -23,35 +27,46 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/flap_kick"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/test_config"
)
var _ = Describe("FlapKick Transformer", func() {
var (
db *postgres.DB
blockChain core.BlockChain
)
BeforeEach(func() {
rpcClient, ethClient, err := getClients(ipc)
Expect(err).NotTo(HaveOccurred())
blockChain, err = getBlockChain(rpcClient, ethClient)
Expect(err).NotTo(HaveOccurred())
db = test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
})
It("fetches and transforms a FlapKick event from Kovan chain", func() {
blockNumber := int64(9002933)
config := flap_kick.FlapKickConfig
config.StartingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber
rpcClient, ethClient, err := getClients(ipc)
Expect(err).NotTo(HaveOccurred())
blockChain, err := getBlockChain(rpcClient, ethClient)
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
db := test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
err = persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
initializer := factories.Transformer{
transformer := factories.Transformer{
Config: config,
Converter: &flap_kick.FlapKickConverter{},
Repository: &flap_kick.FlapKickRepository{},
Fetcher: &shared.Fetcher{},
}
transformer := initializer.NewTransformer(db, blockChain)
err = transformer.Execute()
}.NewTransformer(db)
fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred())
var dbResult []flap_kick.FlapKickModel

19
pkg/transformers/integration_tests/flip_kick.go
View File

@ -65,21 +65,26 @@ var _ = Describe("FlipKick Transformer", func() {
Expect(err).NotTo(HaveOccurred())
blockChain, err := getBlockChain(rpcClient, ethClient)
Expect(err).NotTo(HaveOccurred())
db := test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
err = persistHeader(db, blockNumber, blockChain)
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
initializer := factories.Transformer{
transformer := factories.Transformer{
Config: config,
Converter: &flip_kick.FlipKickConverter{},
Repository: &flip_kick.FlipKickRepository{},
Fetcher: &shared.Fetcher{},
}
transformer := initializer.NewTransformer(db, blockChain)
err = transformer.Execute()
}.NewTransformer(db)
fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred())
var dbResult []flip_kick.FlipKickModel

61
pkg/transformers/integration_tests/flop_kick.go
View File

@ -35,8 +35,13 @@ import (
var _ = Describe("FlopKick Transformer", func() {
var (
db *postgres.DB
blockChain core.BlockChain
db *postgres.DB
blockChain core.BlockChain
config shared.TransformerConfig
initializer factories.Transformer
fetcher shared.LogFetcher
addresses []common.Address
topics []common.Hash
)
BeforeEach(func() {
@ -46,25 +51,33 @@ var _ = Describe("FlopKick Transformer", func() {
Expect(err).NotTo(HaveOccurred())
db = test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
config = flop_kick.Config
initializer = factories.Transformer{
Config: config,
Converter: &flop_kick.FlopKickConverter{},
Repository: &flop_kick.FlopKickRepository{},
}
fetcher = shared.NewFetcher(blockChain)
addresses = shared.HexStringsToAddresses(config.ContractAddresses)
topics = []common.Hash{common.HexToHash(config.Topic)}
})
It("fetches and transforms a FlopKick event from Kovan chain", func() {
blockNumber := int64(8672119)
config := flop_kick.Config
config.StartingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber
initializer.Config.StartingBlockNumber = blockNumber
initializer.Config.EndingBlockNumber = blockNumber
err := persistHeader(db, blockNumber, blockChain)
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
initializer := factories.Transformer{
Config: config,
Converter: &flop_kick.FlopKickConverter{},
Repository: &flop_kick.FlopKickRepository{},
Fetcher: &shared.Fetcher{},
}
transformer := initializer.NewTransformer(db, blockChain)
err = transformer.Execute()
logs, err := fetcher.FetchLogs(addresses, topics, header)
Expect(err).NotTo(HaveOccurred())
transformer := initializer.NewTransformer(db)
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred())
var dbResult []flop_kick.Model
@ -82,21 +95,17 @@ var _ = Describe("FlopKick Transformer", func() {
It("fetches and transforms another FlopKick event from Kovan chain", func() {
blockNumber := int64(8955611)
config := flop_kick.Config
config.StartingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber
initializer.Config.StartingBlockNumber = blockNumber
initializer.Config.EndingBlockNumber = blockNumber
err := persistHeader(db, blockNumber, blockChain)
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
initializer := factories.Transformer{
Config: config,
Converter: &flop_kick.FlopKickConverter{},
Repository: &flop_kick.FlopKickRepository{},
Fetcher: &shared.Fetcher{},
}
transformer := initializer.NewTransformer(db, blockChain)
err = transformer.Execute()
logs, err := fetcher.FetchLogs(addresses, topics, header)
Expect(err).NotTo(HaveOccurred())
transformer := initializer.NewTransformer(db)
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred())
var dbResult []flop_kick.Model

38
pkg/transformers/integration_tests/frob.go
View File

@ -33,8 +33,11 @@ import (
var _ = Describe("Frob Transformer", func() {
var (
db *postgres.DB
blockChain core.BlockChain
db *postgres.DB
blockChain core.BlockChain
fetcher *shared.Fetcher
config shared.TransformerConfig
initializer factories.Transformer
)
BeforeEach(func() {
@ -44,25 +47,32 @@ var _ = Describe("Frob Transformer", func() {
Expect(err).NotTo(HaveOccurred())
db = test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
fetcher = shared.NewFetcher(blockChain)
config = frob.FrobConfig
initializer = factories.Transformer{
Config: config,
Converter: &frob.FrobConverter{},
Repository: &frob.FrobRepository{},
}
})
It("fetches and transforms a Frob event from Kovan chain", func() {
blockNumber := int64(8935258)
config := frob.FrobConfig
config.StartingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber
initializer.Config.StartingBlockNumber = blockNumber
initializer.Config.EndingBlockNumber = blockNumber
err := persistHeader(db, blockNumber, blockChain)
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
initializer := factories.Transformer{
Config: config,
Converter: &frob.FrobConverter{},
Repository: &frob.FrobRepository{},
Fetcher: &shared.Fetcher{},
}
transformer := initializer.NewTransformer(db, blockChain)
err = transformer.Execute()
logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
transformer := initializer.NewTransformer(db)
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred())
var dbResult []frob.FrobModel

10
pkg/transformers/integration_tests/helpers.go
View File

@ -42,12 +42,14 @@ func getBlockChain(rpcClient client.RpcClient, ethClient *ethclient.Client) (cor
return blockChain, nil
}
func persistHeader(db *postgres.DB, blockNumber int64, blockChain core.BlockChain) error {
// Persist the header for a given block to postgres. Returns the header if successful.
func persistHeader(db *postgres.DB, blockNumber int64, blockChain core.BlockChain) (core.Header, error) {
header, err := blockChain.GetHeaderByNumber(blockNumber)
if err != nil {
return err
return core.Header{}, err
}
headerRepository := repositories.NewHeaderRepository(db)
_, err = headerRepository.CreateOrUpdateHeader(header)
return err
id, err := headerRepository.CreateOrUpdateHeader(header)
header.Id = id
return header, err
}

36
pkg/transformers/integration_tests/pit_file_debt_ceiling.go
View File

@ -15,8 +15,11 @@
package integration_tests
import (
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
@ -25,31 +28,44 @@ import (
)
var _ = Describe("PitFileDebtCeiling LogNoteTransformer", func() {
var (
db *postgres.DB
blockChain core.BlockChain
)
BeforeEach(func() {
rpcClient, ethClient, err := getClients(ipc)
Expect(err).NotTo(HaveOccurred())
blockChain, err = getBlockChain(rpcClient, ethClient)
Expect(err).NotTo(HaveOccurred())
db = test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
})
It("fetches and transforms a PitFileDebtCeiling event from Kovan chain", func() {
blockNumber := int64(8535578)
config := debt_ceiling.DebtCeilingFileConfig
config.StartingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber
rpcClient, ethClient, err := getClients(ipc)
Expect(err).NotTo(HaveOccurred())
blockChain, err := getBlockChain(rpcClient, ethClient)
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
db := test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
err = persistHeader(db, blockNumber, blockChain)
fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
initializer := factories.LogNoteTransformer{
Config: config,
Fetcher: &shared.Fetcher{},
Converter: &debt_ceiling.PitFileDebtCeilingConverter{},
Repository: &debt_ceiling.PitFileDebtCeilingRepository{},
}
transformer := initializer.NewLogNoteTransformer(db, blockChain)
err = transformer.Execute()
transformer := initializer.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred())
var dbResult []debt_ceiling.PitFileDebtCeilingModel

72
pkg/transformers/integration_tests/pit_file_ilk.go
View File

@ -15,8 +15,11 @@
package integration_tests
import (
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/ilk"
@ -25,31 +28,47 @@ import (
)
var _ = Describe("PitFileIlk LogNoteTransformer", func() {
It("fetches and transforms a Pit.file ilk 'spot' event from Kovan", func() {
blockNumber := int64(9103223)
config := ilk.IlkFileConfig
config.StartingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber
var (
db *postgres.DB
blockChain core.BlockChain
initializer factories.LogNoteTransformer
addresses []common.Address
topics []common.Hash
)
BeforeEach(func() {
rpcClient, ethClient, err := getClients(ipc)
Expect(err).NotTo(HaveOccurred())
blockChain, err := getBlockChain(rpcClient, ethClient)
blockChain, err = getBlockChain(rpcClient, ethClient)
Expect(err).NotTo(HaveOccurred())
db := test_config.NewTestDB(blockChain.Node())
db = test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
config := ilk.IlkFileConfig
err = persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
addresses = shared.HexStringsToAddresses(config.ContractAddresses)
topics = []common.Hash{common.HexToHash(config.Topic)}
initializer := factories.LogNoteTransformer{
initializer = factories.LogNoteTransformer{
Config: config,
Fetcher: &shared.Fetcher{},
Converter: &ilk.PitFileIlkConverter{},
Repository: &ilk.PitFileIlkRepository{},
}
transformer := initializer.NewLogNoteTransformer(db, blockChain)
err = transformer.Execute()
})
It("fetches and transforms a Pit.file ilk 'spot' event from Kovan", func() {
blockNumber := int64(9103223)
initializer.Config.StartingBlockNumber = blockNumber
initializer.Config.EndingBlockNumber = blockNumber
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(addresses, topics, header)
Expect(err).NotTo(HaveOccurred())
transformer := initializer.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred())
var dbResult []ilk.PitFileIlkModel
@ -64,29 +83,18 @@ var _ = Describe("PitFileIlk LogNoteTransformer", func() {
It("fetches and transforms a Pit.file ilk 'line' event from Kovan", func() {
blockNumber := int64(8762253)
config := ilk.IlkFileConfig
config.StartingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber
initializer.Config.StartingBlockNumber = blockNumber
initializer.Config.EndingBlockNumber = blockNumber
rpcClient, ethClient, err := getClients(ipc)
Expect(err).NotTo(HaveOccurred())
blockChain, err := getBlockChain(rpcClient, ethClient)
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
db := test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
err = persistHeader(db, blockNumber, blockChain)
fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(addresses, topics, header)
Expect(err).NotTo(HaveOccurred())
initializer := factories.LogNoteTransformer{
Config: config,
Fetcher: &shared.Fetcher{},
Converter: &ilk.PitFileIlkConverter{},
Repository: &ilk.PitFileIlkRepository{},
}
transformer := initializer.NewLogNoteTransformer(db, blockChain)
err = transformer.Execute()
transformer := initializer.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred())
var dbResult []ilk.PitFileIlkModel

107
pkg/transformers/integration_tests/price_feeds.go
View File

@ -15,6 +15,7 @@
package integration_tests
import (
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
@ -29,8 +30,12 @@ import (
var _ = Describe("Price feeds transformer", func() {
var (
db *postgres.DB
blockChain core.BlockChain
db *postgres.DB
blockChain core.BlockChain
config shared.TransformerConfig
fetcher *shared.Fetcher
initializer factories.LogNoteTransformer
topics []common.Hash
)
BeforeEach(func() {
@ -40,86 +45,94 @@ var _ = Describe("Price feeds transformer", func() {
Expect(err).NotTo(HaveOccurred())
db = test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
config = price_feeds.PriceFeedConfig
topics = []common.Hash{common.HexToHash(config.Topic)}
fetcher = shared.NewFetcher(blockChain)
initializer = factories.LogNoteTransformer{
Config: config,
Converter: &price_feeds.PriceFeedConverter{},
Repository: &price_feeds.PriceFeedRepository{},
}
})
It("persists a ETH/USD price feed event", func() {
blockNumber := int64(8763054)
err := persistHeader(db, blockNumber, blockChain)
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
config := price_feeds.PriceFeedConfig
config.ContractAddresses = []string{constants.PipContractAddress}
config.StartingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber
addresses := []string{constants.PipContractAddress}
initializer.Config.ContractAddresses = addresses
initializer.Config.StartingBlockNumber = blockNumber
initializer.Config.EndingBlockNumber = blockNumber
transformerInitializer := factories.LogNoteTransformer{
Config: config,
Converter: &price_feeds.PriceFeedConverter{},
Repository: &price_feeds.PriceFeedRepository{},
Fetcher: &shared.Fetcher{},
}
transformer := transformerInitializer.NewLogNoteTransformer(db, blockChain)
logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(addresses),
topics,
header)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute()
transformer := initializer.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred())
var model price_feeds.PriceFeedModel
err = db.Get(&model, `SELECT block_number, medianizer_address, usd_value, tx_idx, raw_log FROM maker.price_feeds WHERE block_number = $1`, config.StartingBlockNumber)
err = db.Get(&model, `SELECT block_number, medianizer_address, usd_value, tx_idx, raw_log FROM maker.price_feeds WHERE block_number = $1`, initializer.Config.StartingBlockNumber)
Expect(err).NotTo(HaveOccurred())
Expect(model.UsdValue).To(Equal("207.314891143000011198"))
Expect(model.MedianizerAddress).To(Equal(config.ContractAddresses[0]))
Expect(model.MedianizerAddress).To(Equal(addresses[0]))
})
It("persists a MKR/USD price feed event", func() {
blockNumber := int64(8763059)
err := persistHeader(db, blockNumber, blockChain)
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
config := price_feeds.PriceFeedConfig
config.ContractAddresses = []string{constants.PepContractAddress}
config.StartingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber
addresses := []string{constants.PepContractAddress}
initializer.Config.ContractAddresses = addresses
initializer.Config.StartingBlockNumber = blockNumber
initializer.Config.EndingBlockNumber = blockNumber
transformerInitializer := factories.LogNoteTransformer{
Config: config,
Converter: &price_feeds.PriceFeedConverter{},
Repository: &price_feeds.PriceFeedRepository{},
Fetcher: &shared.Fetcher{},
}
transformer := transformerInitializer.NewLogNoteTransformer(db, blockChain)
logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(addresses),
topics,
header)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute()
transformer := initializer.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred())
var model price_feeds.PriceFeedModel
err = db.Get(&model, `SELECT block_number, medianizer_address, usd_value, tx_idx, raw_log FROM maker.price_feeds WHERE block_number = $1`, config.StartingBlockNumber)
err = db.Get(&model, `SELECT block_number, medianizer_address, usd_value, tx_idx, raw_log FROM maker.price_feeds WHERE block_number = $1`, initializer.Config.StartingBlockNumber)
Expect(err).NotTo(HaveOccurred())
Expect(model.UsdValue).To(Equal("391.803979212000001553"))
Expect(model.MedianizerAddress).To(Equal(config.ContractAddresses[0]))
Expect(model.MedianizerAddress).To(Equal(addresses[0]))
})
It("persists a REP/USD price feed event", func() {
blockNumber := int64(8763062)
err := persistHeader(db, blockNumber, blockChain)
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
config := price_feeds.PriceFeedConfig
config.ContractAddresses = []string{constants.RepContractAddress}
config.StartingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber
addresses := []string{constants.RepContractAddress}
initializer.Config.ContractAddresses = addresses
initializer.Config.StartingBlockNumber = blockNumber
initializer.Config.EndingBlockNumber = blockNumber
transformerInitializer := factories.LogNoteTransformer{
Config: config,
Converter: &price_feeds.PriceFeedConverter{},
Repository: &price_feeds.PriceFeedRepository{},
Fetcher: &shared.Fetcher{},
}
transformer := transformerInitializer.NewLogNoteTransformer(db, blockChain)
logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(addresses),
topics,
header)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute()
transformer := initializer.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred())
var model price_feeds.PriceFeedModel
err = db.Get(&model, `SELECT block_number, medianizer_address, usd_value, tx_idx, raw_log FROM maker.price_feeds WHERE block_number = $1`, config.StartingBlockNumber)
err = db.Get(&model, `SELECT block_number, medianizer_address, usd_value, tx_idx, raw_log FROM maker.price_feeds WHERE block_number = $1`, initializer.Config.StartingBlockNumber)
Expect(err).NotTo(HaveOccurred())
Expect(model.UsdValue).To(Equal("12.816928482699999847"))
Expect(model.MedianizerAddress).To(Equal(config.ContractAddresses[0]))
Expect(model.MedianizerAddress).To(Equal(addresses[0]))
})
})

82
pkg/transformers/integration_tests/tend.go
View File

@ -15,6 +15,7 @@
package integration_tests
import (
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
@ -28,8 +29,13 @@ import (
var _ = Describe("Tend LogNoteTransformer", func() {
var (
db *postgres.DB
blockChain core.BlockChain
db *postgres.DB
blockChain core.BlockChain
config shared.TransformerConfig
fetcher *shared.Fetcher
initializer factories.LogNoteTransformer
addresses []common.Address
topics []common.Hash
)
BeforeEach(func() {
@ -39,25 +45,33 @@ var _ = Describe("Tend LogNoteTransformer", func() {
Expect(err).NotTo(HaveOccurred())
db = test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
config = tend.TendConfig
fetcher = shared.NewFetcher(blockChain)
addresses = shared.HexStringsToAddresses(config.ContractAddresses)
topics = []common.Hash{common.HexToHash(config.Topic)}
initializer = factories.LogNoteTransformer{
Config: config,
Converter: &tend.TendConverter{},
Repository: &tend.TendRepository{},
}
})
It("fetches and transforms a Flip Tend event from Kovan chain", func() {
blockNumber := int64(8935601)
config := tend.TendConfig
config.StartingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber
initializer.Config.StartingBlockNumber = blockNumber
initializer.Config.EndingBlockNumber = blockNumber
err := persistHeader(db, blockNumber, blockChain)
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
initializer := factories.LogNoteTransformer{
Config: config,
Fetcher: &shared.Fetcher{},
Converter: &tend.TendConverter{},
Repository: &tend.TendRepository{},
}
transformer := initializer.NewLogNoteTransformer(db, blockChain)
err = transformer.Execute()
logs, err := fetcher.FetchLogs(addresses, topics, header)
Expect(err).NotTo(HaveOccurred())
transformer := initializer.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred())
var dbResult []tend.TendModel
@ -80,21 +94,17 @@ var _ = Describe("Tend LogNoteTransformer", func() {
It("fetches and transforms a subsequent Flip Tend event from Kovan chain for the same auction", func() {
blockNumber := int64(8935731)
config := tend.TendConfig
config.StartingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber
initializer.Config.StartingBlockNumber = blockNumber
initializer.Config.EndingBlockNumber = blockNumber
err := persistHeader(db, blockNumber, blockChain)
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
initializer := factories.LogNoteTransformer{
Config: config,
Fetcher: &shared.Fetcher{},
Converter: &tend.TendConverter{},
Repository: &tend.TendRepository{},
}
transformer := initializer.NewLogNoteTransformer(db, blockChain)
err = transformer.Execute()
logs, err := fetcher.FetchLogs(addresses, topics, header)
Expect(err).NotTo(HaveOccurred())
transformer := initializer.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred())
var dbResult []tend.TendModel
@ -117,21 +127,17 @@ var _ = Describe("Tend LogNoteTransformer", func() {
It("fetches and transforms a Flap Tend event from the Kovan chain", func() {
blockNumber := int64(9003177)
config := tend.TendConfig
config.StartingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber
initializer.Config.StartingBlockNumber = blockNumber
initializer.Config.EndingBlockNumber = blockNumber
err := persistHeader(db, blockNumber, blockChain)
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
initializer := factories.LogNoteTransformer{
Config: config,
Fetcher: &shared.Fetcher{},
Converter: &tend.TendConverter{},
Repository: &tend.TendRepository{},
}
transformer := initializer.NewLogNoteTransformer(db, blockChain)
err = transformer.Execute()
logs, err := fetcher.FetchLogs(addresses, topics, header)
Expect(err).NotTo(HaveOccurred())
transformer := initializer.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred())
var dbResult []tend.TendModel

16
pkg/transformers/integration_tests/vat_flux.go
View File

@ -15,6 +15,7 @@
package integration_tests
import (
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
@ -39,17 +40,24 @@ var _ = Describe("VatFlux LogNoteTransformer", func() {
db := test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
err = persistHeader(db, blockNumber, blockChain)
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
initializer := factories.LogNoteTransformer{
Config: config,
Fetcher: &shared.Fetcher{},
Converter: &vat_flux.VatFluxConverter{},
Repository: &vat_flux.VatFluxRepository{},
}
transformer := initializer.NewLogNoteTransformer(db, blockChain)
err = transformer.Execute()
transformer := initializer.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred())
var dbResult []vat_flux.VatFluxModel

40
pkg/transformers/integration_tests/vat_fold.go
View File

@ -18,39 +18,53 @@ import (
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/test_config"
"github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/vat_fold"
"github.com/vulcanize/vulcanizedb/test_config"
)
var _ = Describe("VatFold Transformer", func() {
var (
db *postgres.DB
blockChain core.BlockChain
)
BeforeEach(func() {
rpcClient, ethClient, err := getClients(ipc)
Expect(err).NotTo(HaveOccurred())
blockChain, err = getBlockChain(rpcClient, ethClient)
Expect(err).NotTo(HaveOccurred())
db = test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
})
It("transforms VatFold log events", func() {
blockNumber := int64(9367233)
config := vat_fold.VatFoldConfig
config.StartingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber
rpcClient, ethClient, err := getClients(ipc)
Expect(err).NotTo(HaveOccurred())
blockChain, err := getBlockChain(rpcClient, ethClient)
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
db := test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
err = persistHeader(db, blockNumber, blockChain)
fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
initializer := factories.LogNoteTransformer{
transformer := factories.LogNoteTransformer{
Config: config,
Fetcher: &shared.Fetcher{},
Converter: &vat_fold.VatFoldConverter{},
Repository: &vat_fold.VatFoldRepository{},
}
transformer := initializer.NewLogNoteTransformer(db, blockChain)
err = transformer.Execute()
}.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred())
var dbResults []vat_fold.VatFoldModel

15
pkg/transformers/integration_tests/vat_grab.go
View File

@ -15,6 +15,7 @@
package integration_tests
import (
"github.com/ethereum/go-ethereum/common"
"math/big"
. "github.com/onsi/ginkgo"
@ -41,17 +42,23 @@ var _ = Describe("Vat Grab Transformer", func() {
db := test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
err = persistHeader(db, blockNumber, blockChain)
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
transformer := factories.LogNoteTransformer{
Config: config,
Converter: &vat_grab.VatGrabConverter{},
Repository: &vat_grab.VatGrabRepository{},
Fetcher: &shared.Fetcher{},
}.NewLogNoteTransformer(db, blockChain)
}.NewLogNoteTransformer(db)
err = transformer.Execute()
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred())
var dbResult []vat_grab.VatGrabModel

14
pkg/transformers/integration_tests/vat_heal.go
View File

@ -40,17 +40,23 @@ var _ = Describe("VatHeal Transformer", func() {
db := test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
err = persistHeader(db, blockNumber, blockChain)
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
transformer := factories.LogNoteTransformer{
Config: config,
Converter: &vat_heal.VatHealConverter{},
Repository: &vat_heal.VatHealRepository{},
Fetcher: &shared.Fetcher{},
}.NewLogNoteTransformer(db, blockChain)
}.NewLogNoteTransformer(db)
err = transformer.Execute()
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred())
var dbResults []vat_heal.VatHealModel

19
pkg/transformers/integration_tests/vat_init.go
View File

@ -15,6 +15,7 @@
package integration_tests
import (
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
@ -39,17 +40,23 @@ var _ = Describe("VatInit LogNoteTransformer", func() {
db := test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
err = persistHeader(db, blockNumber, blockChain)
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
initializer := factories.LogNoteTransformer{
fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
transformer := factories.LogNoteTransformer{
Config: config,
Fetcher: &shared.Fetcher{},
Converter: &vat_init.VatInitConverter{},
Repository: &vat_init.VatInitRepository{},
}
transformer := initializer.NewLogNoteTransformer(db, blockChain)
err = transformer.Execute()
}.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred())
var dbResults []vat_init.VatInitModel

18
pkg/transformers/integration_tests/vat_move.go
View File

@ -40,17 +40,23 @@ var _ = Describe("VatMove LogNoteTransformer", func() {
db := test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
err = persistHeader(db, blockNumber, blockChain)
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
initializer := factories.LogNoteTransformer{
fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
transformer := factories.LogNoteTransformer{
Config: config,
Fetcher: &shared.Fetcher{},
Converter: &vat_move.VatMoveConverter{},
Repository: &vat_move.VatMoveRepository{},
}
transformer := initializer.NewLogNoteTransformer(db, blockChain)
err = transformer.Execute()
}.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred())
var dbResults []vat_move.VatMoveModel

22
pkg/transformers/integration_tests/vat_slip.go
View File

@ -1,6 +1,7 @@
package integration_tests
import (
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
@ -18,31 +19,38 @@ var _ = Describe("Vat slip transformer", func() {
blockChain core.BlockChain
)
It("persists vat slip event", func() {
BeforeEach(func() {
rpcClient, ethClient, err := getClients(ipc)
Expect(err).NotTo(HaveOccurred())
blockChain, err = getBlockChain(rpcClient, ethClient)
Expect(err).NotTo(HaveOccurred())
db = test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
})
It("persists vat slip event", func() {
blockNumber := int64(8953655)
config := vat_slip.VatSlipConfig
config.StartingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber
err = persistHeader(db, blockNumber, blockChain)
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
initializer := factories.LogNoteTransformer{
fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
transformer := factories.LogNoteTransformer{
Config: config,
Fetcher: &shared.Fetcher{},
Converter: &vat_slip.VatSlipConverter{},
Repository: &vat_slip.VatSlipRepository{},
}
transformer := initializer.NewLogNoteTransformer(db, blockChain)
}.NewLogNoteTransformer(db)
err = transformer.Execute()
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred())
var headerID int64

19
pkg/transformers/integration_tests/vat_tune.go
View File

@ -15,6 +15,7 @@
package integration_tests
import (
"github.com/ethereum/go-ethereum/common"
"math/big"
. "github.com/onsi/ginkgo"
@ -41,17 +42,23 @@ var _ = Describe("VatTune LogNoteTransformer", func() {
db := test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
err = persistHeader(db, blockNumber, blockChain)
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
initializer := factories.LogNoteTransformer{
fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
transformer := factories.LogNoteTransformer{
Config: config,
Fetcher: &shared.Fetcher{},
Converter: &vat_tune.VatTuneConverter{},
Repository: &vat_tune.VatTuneRepository{},
}
transformer := initializer.NewLogNoteTransformer(db, blockChain)
err = transformer.Execute()
}.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred())
var dbResult []vat_tune.VatTuneModel

40
pkg/transformers/integration_tests/vow_flog.go
View File

@ -15,8 +15,11 @@
package integration_tests
import (
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
@ -25,32 +28,43 @@ import (
)
var _ = Describe("VowFlog LogNoteTransformer", func() {
var (
db *postgres.DB
blockChain core.BlockChain
)
BeforeEach(func() {
rpcClient, ethClient, err := getClients(ipc)
Expect(err).NotTo(HaveOccurred())
blockChain, err = getBlockChain(rpcClient, ethClient)
Expect(err).NotTo(HaveOccurred())
db = test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
})
It("transforms VowFlog log events", func() {
blockNumber := int64(8946819)
config := vow_flog.VowFlogConfig
config.StartingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber
rpcClient, ethClient, err := getClients(ipc)
Expect(err).NotTo(HaveOccurred())
blockChain, err := getBlockChain(rpcClient, ethClient)
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
db := test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
err = persistHeader(db, blockNumber, blockChain)
fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
Expect(1).To(Equal(1))
initializer := factories.LogNoteTransformer{
transformer := factories.LogNoteTransformer{
Config: config,
Fetcher: &shared.Fetcher{},
Converter: &vow_flog.VowFlogConverter{},
Repository: &vow_flog.VowFlogRepository{},
}
transformer := initializer.NewLogNoteTransformer(db, blockChain)
err = transformer.Execute()
}.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred())
var dbResult []vow_flog.VowFlogModel

5
pkg/transformers/pit_file/debt_ceiling/repository.go
View File

@ -16,7 +16,6 @@ package debt_ceiling
import (
"fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -64,10 +63,6 @@ func (repository PitFileDebtCeilingRepository) MarkHeaderChecked(headerID int64)
return shared.MarkHeaderChecked(headerID, repository.db, constants.PitFileDebtCeilingChecked)
}
func (repository PitFileDebtCeilingRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.PitFileDebtCeilingChecked)
}
func (repository *PitFileDebtCeilingRepository) SetDB(db *postgres.DB) {
repository.db = db
}

9
pkg/transformers/pit_file/debt_ceiling/repository_test.go
View File

@ -81,13 +81,4 @@ var _ = Describe("Pit file debt ceiling repository", func() {
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
})
Describe("MissingHeaders", func() {
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
Repository: &pitFileDebtCeilingRepository,
RepositoryTwo: &debt_ceiling.PitFileDebtCeilingRepository{},
}
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
})
})

5
pkg/transformers/pit_file/ilk/repository.go
View File

@ -16,7 +16,6 @@ package ilk
import (
"fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -62,10 +61,6 @@ func (repository PitFileIlkRepository) MarkHeaderChecked(headerID int64) error {
return shared.MarkHeaderChecked(headerID, repository.db, constants.PitFileIlkChecked)
}
func (repository PitFileIlkRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.PitFileIlkChecked)
}
func (repository *PitFileIlkRepository) SetDB(db *postgres.DB) {
repository.db = db
}

9
pkg/transformers/pit_file/ilk/repository_test.go
View File

@ -82,13 +82,4 @@ var _ = Describe("Pit file ilk repository", func() {
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
})
Describe("MissingHeaders", func() {
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
Repository: &pitFileIlkRepository,
RepositoryTwo: &ilk.PitFileIlkRepository{},
}
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
})
})

5
pkg/transformers/price_feeds/repository.go
View File

@ -16,7 +16,6 @@ package price_feeds
import (
"fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -58,10 +57,6 @@ func (repository PriceFeedRepository) MarkHeaderChecked(headerID int64) error {
return shared.MarkHeaderChecked(headerID, repository.db, constants.PriceFeedsChecked)
}
func (repository PriceFeedRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.PriceFeedsChecked)
}
func (repository *PriceFeedRepository) SetDB(db *postgres.DB) {
repository.db = db
}

9
pkg/transformers/price_feeds/repository_test.go
View File

@ -82,13 +82,4 @@ var _ = Describe("Price feeds repository", func() {
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
})
Describe("MissingHeaders", func() {
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
Repository: &priceFeedRepository,
RepositoryTwo: &price_feeds.PriceFeedRepository{},
}
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
})
})

65
pkg/transformers/shared/log_chunker.go Normal file
View File

@ -0,0 +1,65 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package shared
import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
)
type Chunker interface {
AddConfigs(transformerConfigs []TransformerConfig)
ChunkLogs(logs []types.Log) map[string][]types.Log
}
type LogChunker struct {
AddressToNames map[string][]string
NameToTopic0 map[string]common.Hash
}
// Returns a new log chunker with initialised maps.
// Needs to have configs added with `AddConfigs` to consider logs for the respective transformer.
func NewLogChunker() *LogChunker {
return &LogChunker{
AddressToNames: map[string][]string{},
NameToTopic0: map[string]common.Hash{},
}
}
// Configures the chunker by adding more addreses and topics to consider.
func (chunker *LogChunker) AddConfigs(transformerConfigs []TransformerConfig) {
for _, config := range transformerConfigs {
for _, address := range config.ContractAddresses {
chunker.AddressToNames[address] = append(chunker.AddressToNames[address], config.TransformerName)
chunker.NameToTopic0[config.TransformerName] = common.HexToHash(config.Topic)
}
}
}
// Goes through an array of logs, associating relevant logs (matching addresses and topic) with transformers
func (chunker *LogChunker) ChunkLogs(logs []types.Log) map[string][]types.Log {
chunks := map[string][]types.Log{}
for _, log := range logs {
// Topic0 is not unique to each transformer, also need to consider the contract address
relevantTransformers := chunker.AddressToNames[log.Address.String()]
for _, transformer := range relevantTransformers {
if chunker.NameToTopic0[transformer] == log.Topics[0] {
chunks[transformer] = append(chunks[transformer], log)
}
}
}
return chunks
}

137
pkg/transformers/shared/log_chunker_test.go Normal file
View File

@ -0,0 +1,137 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package shared_test
import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
)
var _ = Describe("Log chunker", func() {
var (
configs []shared.TransformerConfig
chunker *shared.LogChunker
)
BeforeEach(func() {
configA := shared.TransformerConfig{
TransformerName: "TransformerA",
ContractAddresses: []string{"0x00000000000000000000000000000000000000A1", "0x00000000000000000000000000000000000000A2"},
Topic: "0xA",
}
configB := shared.TransformerConfig{
TransformerName: "TransformerB",
ContractAddresses: []string{"0x00000000000000000000000000000000000000B1"},
Topic: "0xB",
}
configC := shared.TransformerConfig{
TransformerName: "TransformerC",
ContractAddresses: []string{"0x00000000000000000000000000000000000000A2"},
Topic: "0xC",
}
configs = []shared.TransformerConfig{configA, configB, configC}
chunker = shared.NewLogChunker()
chunker.AddConfigs(configs)
})
Describe("initialisation", func() {
It("creates lookup maps correctly", func() {
Expect(chunker.AddressToNames).To(Equal(map[string][]string{
"0x00000000000000000000000000000000000000A1": []string{"TransformerA"},
"0x00000000000000000000000000000000000000A2": []string{"TransformerA", "TransformerC"},
"0x00000000000000000000000000000000000000B1": []string{"TransformerB"},
}))
Expect(chunker.NameToTopic0).To(Equal(map[string]common.Hash{
"TransformerA": common.HexToHash("0xA"),
"TransformerB": common.HexToHash("0xB"),
"TransformerC": common.HexToHash("0xC"),
}))
})
})
Describe("AddConfigs", func() {
It("can add more configs later", func() {
configD := shared.TransformerConfig{
TransformerName: "TransformerD",
ContractAddresses: []string{"0x000000000000000000000000000000000000000D"},
Topic: "0xD",
}
chunker.AddConfigs([]shared.TransformerConfig{configD})
Expect(chunker.AddressToNames).To(ContainElement([]string{"TransformerD"}))
Expect(chunker.NameToTopic0).To(ContainElement(common.HexToHash("0xD")))
})
})
Describe("ChunkLogs", func() {
It("only associates logs with relevant topic0 and address to transformers", func() {
logs := []types.Log{log1, log2, log3, log4, log5}
chunks := chunker.ChunkLogs(logs)
Expect(chunks["TransformerA"]).To(And(ContainElement(log1), ContainElement(log4)))
Expect(chunks["TransformerB"]).To(BeEmpty())
Expect(chunks["TransformerC"]).To(ContainElement(log5))
})
})
})
var (
// Match TransformerA
log1 = types.Log{
Address: common.HexToAddress("0xA1"),
Topics: []common.Hash{
common.HexToHash("0xA"),
common.HexToHash("0xLogTopic1"),
},
}
// Match TransformerA address, but not topic0
log2 = types.Log{
Address: common.HexToAddress("0xA1"),
Topics: []common.Hash{
common.HexToHash("0xB"),
common.HexToHash("0xLogTopic2"),
},
}
// Match TransformerA topic, but TransformerB address
log3 = types.Log{
Address: common.HexToAddress("0xB1"),
Topics: []common.Hash{
common.HexToHash("0xA"),
common.HexToHash("0xLogTopic3"),
},
}
// Match TransformerA, with the other address
log4 = types.Log{
Address: common.HexToAddress("0xA2"),
Topics: []common.Hash{
common.HexToHash("0xA"),
common.HexToHash("0xLogTopic4"),
},
}
// Match TransformerC, which shares address with TransformerA
log5 = types.Log{
Address: common.HexToAddress("0xA2"),
Topics: []common.Hash{
common.HexToHash("0xC"),
common.HexToHash("0xLogTopic5"),
},
}
)

39
pkg/transformers/shared/log_fetcher.go
View File

@ -23,45 +23,34 @@ import (
)
type LogFetcher interface {
FetchLogs(contractAddresses []string, topics [][]common.Hash, header core.Header) ([]types.Log, error)
}
type SettableLogFetcher interface {
LogFetcher
SetBC(bc core.BlockChain)
FetchLogs(contractAddresses []common.Address, topics []common.Hash, missingHeader core.Header) ([]types.Log, error)
}
type Fetcher struct {
blockChain core.BlockChain
}
func (fetcher *Fetcher) SetBC(bc core.BlockChain) {
fetcher.blockChain = bc
}
func NewFetcher(blockchain core.BlockChain) Fetcher {
return Fetcher{
func NewFetcher(blockchain core.BlockChain) *Fetcher {
return &Fetcher{
blockChain: blockchain,
}
}
func (fetcher Fetcher) FetchLogs(contractAddresses []string, topics [][]common.Hash, header core.Header) ([]types.Log, error) {
addresses := hexStringsToAddresses(contractAddresses)
// Checks all topic0s, on all addresses, fetching matching logs for the given header
func (fetcher Fetcher) FetchLogs(addresses []common.Address, topic0s []common.Hash, header core.Header) ([]types.Log, error) {
blockHash := common.HexToHash(header.Hash)
query := ethereum.FilterQuery{
BlockHash: &blockHash,
Addresses: addresses,
Topics: topics,
}
return fetcher.blockChain.GetEthLogsWithCustomQuery(query)
}
func hexStringsToAddresses(hexStrings []string) []common.Address {
var addresses []common.Address
for _, hexString := range hexStrings {
address := common.HexToAddress(hexString)
addresses = append(addresses, address)
// Search for _any_ of the topics in topic0 position; see docs on `FilterQuery`
Topics: [][]common.Hash{topic0s},
}
return addresses
logs, err := fetcher.blockChain.GetEthLogsWithCustomQuery(query)
if err != nil {
// TODO review aggregate fetching error handling
return []types.Log{}, err
}
return logs, nil
}

12
pkg/transformers/shared/log_fetcher_test.go
View File

@ -32,8 +32,12 @@ var _ = Describe("Fetcher", func() {
fetcher := shared.NewFetcher(blockChain)
header := fakes.FakeHeader
addresses := []string{"0xfakeAddress", "0xanotherFakeAddress"}
topicZeros := [][]common.Hash{{common.BytesToHash([]byte{1, 2, 3, 4, 5})}}
addresses := []common.Address{
common.HexToAddress("0xfakeAddress"),
common.HexToAddress("0xanotherFakeAddress"),
}
topicZeros := []common.Hash{common.BytesToHash([]byte{1, 2, 3, 4, 5})}
_, err := fetcher.FetchLogs(addresses, topicZeros, header)
@ -45,7 +49,7 @@ var _ = Describe("Fetcher", func() {
expectedQuery := ethereum.FilterQuery{
BlockHash: &blockHash,
Addresses: []common.Address{address1, address2},
Topics: topicZeros,
Topics: [][]common.Hash{topicZeros},
}
blockChain.AssertGetEthLogsWithCustomQueryCalledWith(expectedQuery)
})
@ -55,7 +59,7 @@ var _ = Describe("Fetcher", func() {
blockChain.SetGetEthLogsWithCustomQueryErr(fakes.FakeError)
fetcher := shared.NewFetcher(blockChain)
_, err := fetcher.FetchLogs([]string{}, [][]common.Hash{}, core.Header{})
_, err := fetcher.FetchLogs([]common.Address{}, []common.Hash{}, core.Header{})
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))

60
pkg/transformers/shared/repository.go
View File

@ -1,7 +1,10 @@
package shared
import (
"bytes"
"database/sql"
"database/sql/driver"
"fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
@ -22,7 +25,8 @@ func MarkHeaderCheckedInTransaction(headerID int64, tx *sql.Tx, checkedHeadersCo
return err
}
func MissingHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.DB, checkedHeadersColumn string) ([]core.Header, error) {
// Treats a header as missing if it's not in the headers table, or not checked for some log type
func MissingHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.DB, notCheckedSQL string) ([]core.Header, error) {
var result []core.Header
var query string
var err error
@ -30,14 +34,14 @@ func MissingHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.D
if endingBlockNumber == -1 {
query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
LEFT JOIN checked_headers on headers.id = header_id
WHERE (header_id ISNULL OR ` + checkedHeadersColumn + ` IS FALSE)
WHERE (header_id ISNULL OR ` + notCheckedSQL + `)
AND headers.block_number >= $1
AND headers.eth_node_fingerprint = $2`
err = db.Select(&result, query, startingBlockNumber, db.Node.ID)
} else {
query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
LEFT JOIN checked_headers on headers.id = header_id
WHERE (header_id ISNULL OR ` + checkedHeadersColumn + ` IS FALSE)
WHERE (header_id ISNULL OR ` + notCheckedSQL + `)
AND headers.block_number >= $1
AND headers.block_number <= $2
AND headers.eth_node_fingerprint = $3`
@ -46,3 +50,53 @@ func MissingHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.D
return result, err
}
func GetCheckedColumnNames(db *postgres.DB) ([]string, error) {
// Query returns `[]driver.Value`, nullable polymorphic interface
var queryResult []driver.Value
columnNamesQuery :=
`SELECT column_name FROM information_schema.columns
WHERE table_schema = 'public'
AND table_name = 'checked_headers'
AND column_name ~ '_checked';`
err := db.Select(&queryResult, columnNamesQuery)
if err != nil {
return []string{}, err
}
// Transform column names from `driver.Value` to strings
var columnNames []string
for _, result := range queryResult {
if columnName, ok := result.(string); ok {
columnNames = append(columnNames, columnName)
} else {
return []string{}, fmt.Errorf("incorrect value for checked_headers column name")
}
}
return columnNames, nil
}
// Builds a SQL string that checks if any column value is FALSE, given the column names.
// Defaults to FALSE when no columns are provided.
// Ex: ["columnA", "columnB"] => "NOT (columnA AND columnB)"
// [] => "FALSE"
func CreateNotCheckedSQL(boolColumns []string) string {
var result bytes.Buffer
if len(boolColumns) == 0 {
return "FALSE"
}
result.WriteString("NOT (")
// Loop excluding last column name
for _, column := range boolColumns[:len(boolColumns)-1] {
result.WriteString(fmt.Sprintf("%v AND ", column))
}
// No trailing "OR" for the last column name
result.WriteString(fmt.Sprintf("%v)", boolColumns[len(boolColumns)-1]))
return result.String()
}

170
pkg/transformers/shared/repository_utility_test.go Normal file
View File

@ -0,0 +1,170 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package shared_test
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/test_config"
"math/rand"
)
var _ = Describe("Repository utilities", func() {
Describe("MissingHeaders", func() {
var (
db *postgres.DB
headerRepository datastore.HeaderRepository
startingBlockNumber int64
endingBlockNumber int64
eventSpecificBlockNumber int64
blockNumbers []int64
headerIDs []int64
notCheckedSQL string
err error
)
BeforeEach(func() {
db = test_config.NewTestDB(test_config.NewTestNode())
test_config.CleanTestDB(db)
headerRepository = repositories.NewHeaderRepository(db)
columnNames, err := shared.GetCheckedColumnNames(db)
Expect(err).NotTo(HaveOccurred())
notCheckedSQL = shared.CreateNotCheckedSQL(columnNames)
startingBlockNumber = rand.Int63()
eventSpecificBlockNumber = startingBlockNumber + 1
endingBlockNumber = startingBlockNumber + 2
outOfRangeBlockNumber := endingBlockNumber + 1
blockNumbers = []int64{startingBlockNumber, eventSpecificBlockNumber, endingBlockNumber, outOfRangeBlockNumber}
headerIDs = []int64{}
for _, n := range blockNumbers {
headerID, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(n))
headerIDs = append(headerIDs, headerID)
Expect(err).NotTo(HaveOccurred())
}
})
It("only treats headers as checked if the event specific logs have been checked", func() {
_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
Expect(err).NotTo(HaveOccurred())
headers, err := shared.MissingHeaders(startingBlockNumber, endingBlockNumber, db, notCheckedSQL)
Expect(err).NotTo(HaveOccurred())
Expect(len(headers)).To(Equal(3))
Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber)))
Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber)))
Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber)))
})
It("only returns headers associated with the current node", func() {
dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
for _, n := range blockNumbers {
_, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n + 10))
Expect(err).NotTo(HaveOccurred())
}
Expect(err).NotTo(HaveOccurred())
nodeOneMissingHeaders, err := shared.MissingHeaders(startingBlockNumber, endingBlockNumber, db, notCheckedSQL)
Expect(err).NotTo(HaveOccurred())
Expect(len(nodeOneMissingHeaders)).To(Equal(3))
Expect(nodeOneMissingHeaders[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(eventSpecificBlockNumber), Equal(endingBlockNumber)))
Expect(nodeOneMissingHeaders[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(eventSpecificBlockNumber), Equal(endingBlockNumber)))
Expect(nodeOneMissingHeaders[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(startingBlockNumber), Equal(eventSpecificBlockNumber), Equal(endingBlockNumber)))
nodeTwoMissingHeaders, err := shared.MissingHeaders(startingBlockNumber, endingBlockNumber+10, dbTwo, notCheckedSQL)
Expect(err).NotTo(HaveOccurred())
Expect(len(nodeTwoMissingHeaders)).To(Equal(3))
Expect(nodeTwoMissingHeaders[0].BlockNumber).To(Or(Equal(startingBlockNumber+10), Equal(eventSpecificBlockNumber+10), Equal(endingBlockNumber+10)))
Expect(nodeTwoMissingHeaders[1].BlockNumber).To(Or(Equal(startingBlockNumber+10), Equal(eventSpecificBlockNumber+10), Equal(endingBlockNumber+10)))
})
})
Describe("GetCheckedColumnNames", func() {
It("gets the column names from checked_headers", func() {
db := test_config.NewTestDB(test_config.NewTestNode())
test_config.CleanTestDB(db)
expectedColumnNames := getExpectedColumnNames()
actualColumnNames, err := shared.GetCheckedColumnNames(db)
Expect(err).NotTo(HaveOccurred())
Expect(actualColumnNames).To(Equal(expectedColumnNames))
})
})
Describe("CreateNotCheckedSQL", func() {
It("generates a correct SQL string for one column", func() {
columns := []string{"columnA"}
expected := "NOT (columnA)"
actual := shared.CreateNotCheckedSQL(columns)
Expect(actual).To(Equal(expected))
})
It("generates a correct SQL string for several columns", func() {
columns := []string{"columnA", "columnB"}
expected := "NOT (columnA AND columnB)"
actual := shared.CreateNotCheckedSQL(columns)
Expect(actual).To(Equal(expected))
})
It("defaults to FALSE when there are no columns", func() {
expected := "FALSE"
actual := shared.CreateNotCheckedSQL([]string{})
Expect(actual).To(Equal(expected))
})
})
})
func getExpectedColumnNames() []string {
return []string{
"price_feeds_checked",
"flip_kick_checked",
"frob_checked",
"tend_checked",
"bite_checked",
"dent_checked",
"pit_file_debt_ceiling_checked",
"pit_file_ilk_checked",
"vat_init_checked",
"drip_file_ilk_checked",
"drip_file_repo_checked",
"drip_file_vow_checked",
"deal_checked",
"drip_drip_checked",
"cat_file_chop_lump_checked",
"cat_file_flip_checked",
"cat_file_pit_vow_checked",
"flop_kick_checked",
"vat_move_checked",
"vat_fold_checked",
"vat_heal_checked",
"vat_toll_checked",
"vat_tune_checked",
"vat_grab_checked",
"vat_flux_checked",
"vat_slip_checked",
"vow_flog_checked",
"flap_kick_checked",
}
}

15
pkg/transformers/shared/transformer.go
View File

@ -16,16 +16,18 @@ package shared
import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
type Transformer interface {
Execute() error
Execute(logs []types.Log, header core.Header) error
GetConfig() TransformerConfig
}
type TransformerInitializer func(db *postgres.DB, blockChain core.BlockChain) Transformer
type TransformerInitializer func(db *postgres.DB) Transformer
type TransformerConfig struct {
TransformerName string
@ -45,3 +47,10 @@ func HexToString(byteString string) string {
value := common.HexToHash(byteString)
return value.Big().String()
}
func HexStringsToAddresses(strings []string) (addresses []common.Address) {
for _, hexString := range strings {
addresses = append(addresses, common.HexToAddress(hexString))
}
return
}

5
pkg/transformers/tend/repository.go
View File

@ -16,7 +16,6 @@ package tend
import (
"fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -68,10 +67,6 @@ func (repository TendRepository) MarkHeaderChecked(headerId int64) error {
return shared.MarkHeaderChecked(headerId, repository.db, constants.TendChecked)
}
func (repository TendRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.TendChecked)
}
func (repository *TendRepository) SetDB(db *postgres.DB) {
repository.db = db
}

9
pkg/transformers/tend/repository_test.go
View File

@ -94,13 +94,4 @@ var _ = Describe("TendRepository", func() {
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
})
Describe("MissingHeaders", func() {
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
Repository: &tendRepository,
RepositoryTwo: &tend.TendRepository{},
}
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
})
})

8
pkg/transformers/test_data/mocks/log_fetcher.go
View File

@ -21,18 +21,20 @@ import (
)
type MockLogFetcher struct {
FetchedContractAddresses [][]string
FetchedContractAddresses [][]common.Address
FetchedTopics [][]common.Hash
FetchedBlocks []int64
fetcherError error
FetchedLogs []types.Log
SetBcCalled bool
FetchLogsCalled bool
}
func (mlf *MockLogFetcher) FetchLogs(contractAddresses []string, topics [][]common.Hash, header core.Header) ([]types.Log, error) {
func (mlf *MockLogFetcher) FetchLogs(contractAddresses []common.Address, topics []common.Hash, header core.Header) ([]types.Log, error) {
mlf.FetchedContractAddresses = append(mlf.FetchedContractAddresses, contractAddresses)
mlf.FetchedTopics = topics
mlf.FetchedTopics = [][]common.Hash{topics}
mlf.FetchedBlocks = append(mlf.FetchedBlocks, header.BlockNumber)
mlf.FetchLogsCalled = true
return mlf.FetchedLogs, mlf.fetcherError
}

53
pkg/transformers/test_data/mocks/mock_watcher_repository.go Normal file
View File

@ -0,0 +1,53 @@
package mocks
import (
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
type MockWatcherRepository struct {
ReturnCheckedColumnNames []string
GetCheckedColumnNamesError error
GetCheckedColumnNamesCalled bool
ReturnNotCheckedSQL string
CreateNotCheckedSQLCalled bool
ReturnMissingHeaders []core.Header
MissingHeadersError error
MissingHeadersCalled bool
}
func (repository *MockWatcherRepository) GetCheckedColumnNames(db *postgres.DB) ([]string, error) {
repository.GetCheckedColumnNamesCalled = true
if repository.GetCheckedColumnNamesError != nil {
return []string{}, repository.GetCheckedColumnNamesError
}
return repository.ReturnCheckedColumnNames, nil
}
func (repository *MockWatcherRepository) SetCheckedColumnNames(checkedColumnNames []string) {
repository.ReturnCheckedColumnNames = checkedColumnNames
}
func (repository *MockWatcherRepository) CreateNotCheckedSQL(boolColumns []string) string {
repository.CreateNotCheckedSQLCalled = true
return repository.ReturnNotCheckedSQL
}
func (repository *MockWatcherRepository) SetNotCheckedSQL(notCheckedSql string) {
repository.ReturnNotCheckedSQL = notCheckedSql
}
func (repository *MockWatcherRepository) MissingHeaders(startingBlockNumber int64, endingBlockNumber int64, db *postgres.DB, notCheckedSQL string) ([]core.Header, error) {
if repository.MissingHeadersError != nil {
return []core.Header{}, repository.MissingHeadersError
}
repository.MissingHeadersCalled = true
return repository.ReturnMissingHeaders, nil
}
func (repository *MockWatcherRepository) SetMissingHeaders(headers []core.Header) {
repository.ReturnMissingHeaders = headers
}

17
pkg/transformers/test_data/mocks/repository.go
View File

@ -3,7 +3,6 @@ package mocks
import (
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
@ -12,8 +11,6 @@ type MockRepository struct {
markHeaderCheckedError error
MarkHeaderCheckedPassedHeaderIDs []int64
CreatedHeaderIds []int64
missingHeaders []core.Header
missingHeadersError error
PassedStartingBlockNumber int64
PassedEndingBlockNumber int64
PassedHeaderID int64
@ -36,24 +33,10 @@ func (repository *MockRepository) MarkHeaderChecked(headerID int64) error {
return repository.markHeaderCheckedError
}
func (repository *MockRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
repository.PassedStartingBlockNumber = startingBlockNumber
repository.PassedEndingBlockNumber = endingBlockNumber
return repository.missingHeaders, repository.missingHeadersError
}
func (repository *MockRepository) SetDB(db *postgres.DB) {
repository.SetDbCalled = true
}
func (repository *MockRepository) SetMissingHeadersError(e error) {
repository.missingHeadersError = e
}
func (repository *MockRepository) SetMissingHeaders(headers []core.Header) {
repository.missingHeaders = headers
}
func (repository *MockRepository) SetMarkHeaderCheckedError(e error) {
repository.markHeaderCheckedError = e
}

44
pkg/transformers/test_data/mocks/transformer.go Normal file
View File

@ -0,0 +1,44 @@
package mocks
import (
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
)
type MockTransformer struct {
ExecuteWasCalled bool
ExecuteError error
PassedLogs []types.Log
PassedHeader core.Header
config shared.TransformerConfig
}
func (mh *MockTransformer) Execute(logs []types.Log, header core.Header) error {
if mh.ExecuteError != nil {
return mh.ExecuteError
}
mh.ExecuteWasCalled = true
mh.PassedLogs = logs
mh.PassedHeader = header
return nil
}
func (mh *MockTransformer) GetConfig() shared.TransformerConfig {
return mh.config
}
func (mh *MockTransformer) SetTransformerConfig(config shared.TransformerConfig) {
mh.config = config
}
func (mh *MockTransformer) FakeTransformerInitializer(db *postgres.DB) shared.Transformer {
return mh
}
var FakeTransformerConfig = shared.TransformerConfig{
TransformerName: "FakeTransformer",
ContractAddresses: []string{"FakeAddress"},
Topic: "FakeTopic",
}

89
pkg/transformers/test_data/shared_behaviors/repository_behaviors.go
View File

@ -17,7 +17,6 @@ package shared_behaviors
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
@ -25,7 +24,6 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
"github.com/vulcanize/vulcanizedb/test_config"
"math/rand"
)
var (
@ -61,7 +59,7 @@ func SharedRepositoryCreateBehaviors(inputs *CreateBehaviorInputs) {
var logEventModel = inputs.TestModel
BeforeEach(func() {
headerRepository = GetHeaderRepository()
headerRepository = getHeaderRepository()
headerID, err = headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
Expect(err).NotTo(HaveOccurred())
})
@ -132,94 +130,13 @@ func SharedRepositoryCreateBehaviors(inputs *CreateBehaviorInputs) {
})
}
func SharedRepositoryMissingHeadersBehaviors(inputs *MissingHeadersBehaviorInputs) {
Describe("MissingHeaders", func() {
var (
repository = inputs.Repository
startingBlockNumber int64
endingBlockNumber int64
eventSpecificBlockNumber int64
blockNumbers []int64
headerIDs []int64
)
BeforeEach(func() {
headerRepository = GetHeaderRepository()
startingBlockNumber = rand.Int63()
eventSpecificBlockNumber = startingBlockNumber + 1
endingBlockNumber = startingBlockNumber + 2
outOfRangeBlockNumber := endingBlockNumber + 1
blockNumbers = []int64{startingBlockNumber, eventSpecificBlockNumber, endingBlockNumber, outOfRangeBlockNumber}
headerIDs = []int64{}
for _, n := range blockNumbers {
headerID, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(n))
headerIDs = append(headerIDs, headerID)
Expect(err).NotTo(HaveOccurred())
}
})
It("returns headers that haven't been checked", func() {
err := repository.MarkHeaderChecked(headerIDs[1])
Expect(err).NotTo(HaveOccurred())
headers, err := repository.MissingHeaders(startingBlockNumber, endingBlockNumber)
Expect(err).NotTo(HaveOccurred())
Expect(len(headers)).To(Equal(2))
Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
})
It("only treats headers as checked if the event specific logs have been checked", func() {
_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
Expect(err).NotTo(HaveOccurred())
headers, err := repository.MissingHeaders(startingBlockNumber, endingBlockNumber)
Expect(err).NotTo(HaveOccurred())
Expect(len(headers)).To(Equal(3))
Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber)))
Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber)))
Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber)))
})
It("only returns headers associated with the current node", func() {
dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
for _, n := range blockNumbers {
_, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n))
Expect(err).NotTo(HaveOccurred())
}
repositoryTwo := inputs.RepositoryTwo
repositoryTwo.SetDB(dbTwo)
err := repository.MarkHeaderChecked(headerIDs[0])
Expect(err).NotTo(HaveOccurred())
nodeOneMissingHeaders, err := repository.MissingHeaders(startingBlockNumber, endingBlockNumber)
Expect(err).NotTo(HaveOccurred())
Expect(len(nodeOneMissingHeaders)).To(Equal(2))
Expect(nodeOneMissingHeaders[0].BlockNumber).To(Or(Equal(eventSpecificBlockNumber), Equal(endingBlockNumber)))
Expect(nodeOneMissingHeaders[1].BlockNumber).To(Or(Equal(eventSpecificBlockNumber), Equal(endingBlockNumber)))
nodeTwoMissingHeaders, err := repositoryTwo.MissingHeaders(startingBlockNumber, endingBlockNumber)
Expect(err).NotTo(HaveOccurred())
Expect(len(nodeTwoMissingHeaders)).To(Equal(3))
Expect(nodeTwoMissingHeaders[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(eventSpecificBlockNumber), Equal(endingBlockNumber)))
Expect(nodeTwoMissingHeaders[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(eventSpecificBlockNumber), Equal(endingBlockNumber)))
Expect(nodeTwoMissingHeaders[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(eventSpecificBlockNumber), Equal(endingBlockNumber)))
})
})
}
func SharedRepositoryMarkHeaderCheckedBehaviors(inputs *MarkedHeaderCheckedBehaviorInputs) {
var repository = inputs.Repository
var checkedHeaderColumn = inputs.CheckedHeaderColumnName
Describe("MarkHeaderChecked", func() {
BeforeEach(func() {
headerRepository = GetHeaderRepository()
headerRepository = getHeaderRepository()
headerId, err = headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
Expect(err).NotTo(HaveOccurred())
})
@ -257,7 +174,7 @@ func SharedRepositoryMarkHeaderCheckedBehaviors(inputs *MarkedHeaderCheckedBehav
})
}
func GetHeaderRepository() repositories.HeaderRepository {
func getHeaderRepository() repositories.HeaderRepository {
db = test_config.NewTestDB(test_config.NewTestNode())
test_config.CleanTestDB(db)

298
pkg/transformers/transformers.go
View File

@ -47,233 +47,223 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/transformers/vow_flog"
)
// Custom event transformers
var (
BiteTransformerInitializer = factories.Transformer{
BiteTransformer = factories.Transformer{
Config: bite.BiteConfig,
Converter: &bite.BiteConverter{},
Repository: &bite.BiteRepository{},
Fetcher: &shared.Fetcher{},
}.NewTransformer
}
CatFileChopLumpTransformerInitializer = factories.LogNoteTransformer{
Config: chop_lump.CatFileChopLumpConfig,
Converter: &chop_lump.CatFileChopLumpConverter{},
Repository: &chop_lump.CatFileChopLumpRepository{},
Fetcher: &shared.Fetcher{},
}.NewLogNoteTransformer
CatFileFlipTransformerInitializer = factories.LogNoteTransformer{
Config: flip.CatFileFlipConfig,
Converter: &flip.CatFileFlipConverter{},
Repository: &flip.CatFileFlipRepository{},
Fetcher: &shared.Fetcher{},
}.NewLogNoteTransformer
CatFilePitVowTransformerInitializer = factories.LogNoteTransformer{
Config: pit_vow.CatFilePitVowConfig,
Converter: &pit_vow.CatFilePitVowConverter{},
Repository: &pit_vow.CatFilePitVowRepository{},
Fetcher: &shared.Fetcher{},
}.NewLogNoteTransformer
DealTransformerInitializer = factories.LogNoteTransformer{
Config: deal.DealConfig,
Converter: &deal.DealConverter{},
Repository: &deal.DealRepository{},
Fetcher: &shared.Fetcher{},
}.NewLogNoteTransformer
DentTransformerInitializer = factories.LogNoteTransformer{
Config: dent.DentConfig,
Converter: &dent.DentConverter{},
Repository: &dent.DentRepository{},
Fetcher: &shared.Fetcher{},
}.NewLogNoteTransformer
DripDripTransformerInitializer = factories.LogNoteTransformer{
Config: drip_drip.DripDripConfig,
Converter: &drip_drip.DripDripConverter{},
Repository: &drip_drip.DripDripRepository{},
Fetcher: &shared.Fetcher{},
}.NewLogNoteTransformer
DripFileIlkTransformerInitializer = factories.LogNoteTransformer{
Config: ilk2.DripFileIlkConfig,
Converter: &ilk2.DripFileIlkConverter{},
Repository: &ilk2.DripFileIlkRepository{},
Fetcher: &shared.Fetcher{},
}.NewLogNoteTransformer
DripFileRepoTransformerInitializer = factories.LogNoteTransformer{
Config: repo.DripFileRepoConfig,
Converter: &repo.DripFileRepoConverter{},
Repository: &repo.DripFileRepoRepository{},
Fetcher: &shared.Fetcher{},
}.NewLogNoteTransformer
DripFileVowTransfromerInitializer = factories.LogNoteTransformer{
Config: vow.DripFileVowConfig,
Converter: &vow.DripFileVowConverter{},
Repository: &vow.DripFileVowRepository{},
Fetcher: &shared.Fetcher{},
}.NewLogNoteTransformer
FlapKickTransformerInitializer = factories.Transformer{
FlapKickTransformer = factories.Transformer{
Config: flap_kick.FlapKickConfig,
Converter: &flap_kick.FlapKickConverter{},
Repository: &flap_kick.FlapKickRepository{},
Fetcher: &shared.Fetcher{},
}.NewTransformer
}
FlipKickTransformerInitializer = factories.Transformer{
FlipKickTransformer = factories.Transformer{
Config: flip_kick.FlipKickConfig,
Converter: &flip_kick.FlipKickConverter{},
Repository: &flip_kick.FlipKickRepository{},
Fetcher: &shared.Fetcher{},
}.NewTransformer
}
FlogTransformerInitializer = factories.LogNoteTransformer{
Config: vow_flog.VowFlogConfig,
Converter: &vow_flog.VowFlogConverter{},
Repository: &vow_flog.VowFlogRepository{},
Fetcher: &shared.Fetcher{},
}.NewLogNoteTransformer
FrobTransformerInitializer = factories.Transformer{
FrobTransformer = factories.Transformer{
Config: frob.FrobConfig,
Converter: &frob.FrobConverter{},
Repository: &frob.FrobRepository{},
Fetcher: &shared.Fetcher{},
}.NewTransformer
}
FlopKickTransformerInitializer = factories.Transformer{
FlopKickTransformer = factories.Transformer{
Config: flop_kick.Config,
Converter: &flop_kick.FlopKickConverter{},
Repository: &flop_kick.FlopKickRepository{},
Fetcher: &shared.Fetcher{},
}.NewTransformer
}
PitFileDebtCeilingTransformerInitializer = factories.LogNoteTransformer{
customEventTransformers = []factories.Transformer{
BiteTransformer,
FlapKickTransformer,
FlipKickTransformer,
FrobTransformer,
FlopKickTransformer,
}
)
// LogNote transformers
var (
CatFileChopLumpTransformer = factories.LogNoteTransformer{
Config: chop_lump.CatFileChopLumpConfig,
Converter: &chop_lump.CatFileChopLumpConverter{},
Repository: &chop_lump.CatFileChopLumpRepository{},
}
CatFileFlipTransformer = factories.LogNoteTransformer{
Config: flip.CatFileFlipConfig,
Converter: &flip.CatFileFlipConverter{},
Repository: &flip.CatFileFlipRepository{},
}
CatFilePitVowTransformer = factories.LogNoteTransformer{
Config: pit_vow.CatFilePitVowConfig,
Converter: &pit_vow.CatFilePitVowConverter{},
Repository: &pit_vow.CatFilePitVowRepository{},
}
DealTransformer = factories.LogNoteTransformer{
Config: deal.DealConfig,
Converter: &deal.DealConverter{},
Repository: &deal.DealRepository{},
}
DentTransformer = factories.LogNoteTransformer{
Config: dent.DentConfig,
Converter: &dent.DentConverter{},
Repository: &dent.DentRepository{},
}
DripDripTransformer = factories.LogNoteTransformer{
Config: drip_drip.DripDripConfig,
Converter: &drip_drip.DripDripConverter{},
Repository: &drip_drip.DripDripRepository{},
}
DripFileIlkTransformer = factories.LogNoteTransformer{
Config: ilk2.DripFileIlkConfig,
Converter: &ilk2.DripFileIlkConverter{},
Repository: &ilk2.DripFileIlkRepository{},
}
DripFileRepoTransformer = factories.LogNoteTransformer{
Config: repo.DripFileRepoConfig,
Converter: &repo.DripFileRepoConverter{},
Repository: &repo.DripFileRepoRepository{},
}
DripFileVowTransfromer = factories.LogNoteTransformer{
Config: vow.DripFileVowConfig,
Converter: &vow.DripFileVowConverter{},
Repository: &vow.DripFileVowRepository{},
}
FlogTransformer = factories.LogNoteTransformer{
Config: vow_flog.VowFlogConfig,
Converter: &vow_flog.VowFlogConverter{},
Repository: &vow_flog.VowFlogRepository{},
}
PitFileDebtCeilingTransformer = factories.LogNoteTransformer{
Config: debt_ceiling.DebtCeilingFileConfig,
Converter: &debt_ceiling.PitFileDebtCeilingConverter{},
Repository: &debt_ceiling.PitFileDebtCeilingRepository{},
Fetcher: &shared.Fetcher{},
}.NewLogNoteTransformer
}
PitFileIlkTransformerInitializer = factories.LogNoteTransformer{
PitFileIlkTransformer = factories.LogNoteTransformer{
Config: ilk.IlkFileConfig,
Converter: &ilk.PitFileIlkConverter{},
Repository: &ilk.PitFileIlkRepository{},
Fetcher: &shared.Fetcher{},
}.NewLogNoteTransformer
}
PriceFeedTransformerInitializer = factories.LogNoteTransformer{
PriceFeedTransformer = factories.LogNoteTransformer{
Config: price_feeds.PriceFeedConfig,
Converter: &price_feeds.PriceFeedConverter{},
Repository: &price_feeds.PriceFeedRepository{},
Fetcher: &shared.Fetcher{},
}.NewLogNoteTransformer
}
TendTransformerInitializer = factories.LogNoteTransformer{
TendTransformer = factories.LogNoteTransformer{
Config: tend.TendConfig,
Converter: &tend.TendConverter{},
Repository: &tend.TendRepository{},
Fetcher: &shared.Fetcher{},
}.NewLogNoteTransformer
}
VatInitTransformerInitializer = factories.LogNoteTransformer{
VatInitTransformer = factories.LogNoteTransformer{
Config: vat_init.VatInitConfig,
Converter: &vat_init.VatInitConverter{},
Repository: &vat_init.VatInitRepository{},
Fetcher: &shared.Fetcher{},
}.NewLogNoteTransformer
}
VatGrabTransformerInitializer = factories.LogNoteTransformer{
VatGrabTransformer = factories.LogNoteTransformer{
Config: vat_grab.VatGrabConfig,
Converter: &vat_grab.VatGrabConverter{},
Repository: &vat_grab.VatGrabRepository{},
Fetcher: &shared.Fetcher{},
}.NewLogNoteTransformer
}
VatFoldTransformerInitializer = factories.LogNoteTransformer{
VatFoldTransformer = factories.LogNoteTransformer{
Config: vat_fold.VatFoldConfig,
Converter: &vat_fold.VatFoldConverter{},
Repository: &vat_fold.VatFoldRepository{},
Fetcher: &shared.Fetcher{},
}.NewLogNoteTransformer
}
VatHealTransformerInitializer = factories.LogNoteTransformer{
VatHealTransformer = factories.LogNoteTransformer{
Config: vat_heal.VatHealConfig,
Converter: &vat_heal.VatHealConverter{},
Repository: &vat_heal.VatHealRepository{},
Fetcher: &shared.Fetcher{},
}.NewLogNoteTransformer
}
VatMoveTransformerInitializer = factories.LogNoteTransformer{
VatMoveTransformer = factories.LogNoteTransformer{
Config: vat_move.VatMoveConfig,
Converter: &vat_move.VatMoveConverter{},
Repository: &vat_move.VatMoveRepository{},
Fetcher: &shared.Fetcher{},
}.NewLogNoteTransformer
}
VatSlipTransformerInitializer = factories.LogNoteTransformer{
VatSlipTransformer = factories.LogNoteTransformer{
Config: vat_slip.VatSlipConfig,
Converter: &vat_slip.VatSlipConverter{},
Repository: &vat_slip.VatSlipRepository{},
Fetcher: &shared.Fetcher{},
}.NewLogNoteTransformer
}
VatTollTransformerInitializer = factories.LogNoteTransformer{
VatTollTransformer = factories.LogNoteTransformer{
Config: vat_toll.VatTollConfig,
Converter: &vat_toll.VatTollConverter{},
Repository: &vat_toll.VatTollRepository{},
Fetcher: &shared.Fetcher{},
}.NewLogNoteTransformer
}
VatTuneTransformerInitializer = factories.LogNoteTransformer{
VatTuneTransformer = factories.LogNoteTransformer{
Config: vat_tune.VatTuneConfig,
Converter: &vat_tune.VatTuneConverter{},
Repository: &vat_tune.VatTuneRepository{},
Fetcher: &shared.Fetcher{},
}.NewLogNoteTransformer
}
VatFluxTransformerInitializer = factories.LogNoteTransformer{
VatFluxTransformer = factories.LogNoteTransformer{
Config: vat_flux.VatFluxConfig,
Converter: &vat_flux.VatFluxConverter{},
Repository: &vat_flux.VatFluxRepository{},
Fetcher: &shared.Fetcher{},
}.NewLogNoteTransformer
}
logNoteTransformers = []factories.LogNoteTransformer{
CatFileChopLumpTransformer,
CatFileFlipTransformer,
CatFilePitVowTransformer,
DealTransformer,
DentTransformer,
DripDripTransformer,
DripFileIlkTransformer,
DripFileRepoTransformer,
DripFileVowTransfromer,
FlogTransformer,
PitFileDebtCeilingTransformer,
PitFileIlkTransformer,
PriceFeedTransformer,
TendTransformer,
VatInitTransformer,
VatGrabTransformer,
VatFoldTransformer,
VatHealTransformer,
VatMoveTransformer,
VatSlipTransformer,
VatTollTransformer,
VatTuneTransformer,
VatFluxTransformer,
}
)
func TransformerInitializers() []shared.TransformerInitializer {
return []shared.TransformerInitializer{
BiteTransformerInitializer,
CatFileChopLumpTransformerInitializer,
CatFileFlipTransformerInitializer,
CatFilePitVowTransformerInitializer,
DealTransformerInitializer,
DentTransformerInitializer,
DripDripTransformerInitializer,
DripFileIlkTransformerInitializer,
DripFileVowTransfromerInitializer,
DripFileRepoTransformerInitializer,
FlapKickTransformerInitializer,
FlipKickTransformerInitializer,
FlogTransformerInitializer,
FlopKickTransformerInitializer,
FrobTransformerInitializer,
PitFileDebtCeilingTransformerInitializer,
PitFileIlkTransformerInitializer,
PriceFeedTransformerInitializer,
TendTransformerInitializer,
VatGrabTransformerInitializer,
VatInitTransformerInitializer,
VatMoveTransformerInitializer,
VatHealTransformerInitializer,
VatFoldTransformerInitializer,
VatSlipTransformerInitializer,
VatTollTransformerInitializer,
VatTuneTransformerInitializer,
VatFluxTransformerInitializer,
// `TransformerInitializers` returns a list of functions, that given a db pointer
// will return a `shared.Transformer`
func TransformerInitializers() (initializers []shared.TransformerInitializer) {
for _, transformer := range logNoteTransformers {
initializers = append(initializers, transformer.NewLogNoteTransformer)
}
for _, transformer := range customEventTransformers {
initializers = append(initializers, transformer.NewTransformer)
}
return
}

Some files were not shown because too many files have changed in this diff Show More