cerc-io/ipld-eth-server
18
0
Fork 1
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases 37 Wiki Activity

Merge pull request #128 from 8thlight/VDB-125-aggregate-log-fetching

Browse Source 
VDB-125 Aggregate log fetching
This commit is contained in:
Edvard Hübinette 2019-01-08 10:51:42 +01:00 committed by GitHub
commit 77209d3b62
120 changed files with 1642 additions and 2574 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
cmd/backfillMakerLogs.go
View File

@ -17,7 +17,6 @@ package cmd
import ( import (
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/vulcanize/vulcanizedb/libraries/shared" "github.com/vulcanize/vulcanizedb/libraries/shared"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers" "github.com/vulcanize/vulcanizedb/pkg/transformers"
@ -46,13 +45,13 @@ func backfillMakerLogs() {
log.Fatal("Failed to initialize database.") log.Fatal("Failed to initialize database.")
} }
watcher := shared.Watcher{ watcher := shared.NewWatcher(db, blockChain)
DB: *db,
Blockchain: blockChain,
}
watcher.AddTransformers(transformers.TransformerInitializers()) watcher.AddTransformers(transformers.TransformerInitializers())
watcher.Execute() err = watcher.Execute()
if err != nil {
// TODO Handle watcher error in backfillMakerLogs
}
} }
func init() { func init() {

90
cmd/continuousLogSync.go
View File

@ -58,70 +58,66 @@ func syncMakerLogs() {
log.Fatal("Failed to initialize database.") log.Fatal("Failed to initialize database.")
} }
watcher := shared.Watcher{ initializers := getTransformerInitializers(transformerNames)
DB: *db,
Blockchain: blockChain,
}
transformerInititalizers := getTransformerInititalizers(transformerNames) watcher := shared.NewWatcher(db, blockChain)
watcher.AddTransformers(transformerInititalizers) watcher.AddTransformers(initializers)
for range ticker.C { for range ticker.C {
watcher.Execute() err = watcher.Execute()
if err != nil {
// TODO Handle watcher errors in ContinuousLogSync
}
} }
} }
func getTransformerInititalizers(transformerNames []string) []shared2.TransformerInitializer { func getTransformerInitializers(transformerNames []string) []shared2.TransformerInitializer {
transformerInitializerMap := buildTransformerInitializerMap() var initializers []shared2.TransformerInitializer
var transformerInitializers []shared2.TransformerInitializer
if transformerNames[0] == "all" { if transformerNames[0] == "all" {
for _, v := range transformerInitializerMap { initializers = transformers.TransformerInitializers()
transformerInitializers = append(transformerInitializers, v)
}
} else { } else {
initializerMap := buildTransformerInitializerMap()
for _, transformerName := range transformerNames { for _, transformerName := range transformerNames {
initializer := transformerInitializerMap[transformerName] initializers = append(initializers, initializerMap[transformerName])
transformerInitializers = append(transformerInitializers, initializer)
} }
} }
return initializers
return transformerInitializers
} }
func buildTransformerInitializerMap() map[string]shared2.TransformerInitializer { func buildTransformerInitializerMap() map[string]shared2.TransformerInitializer {
transformerInitializerMap := make(map[string]shared2.TransformerInitializer) initializerMap := make(map[string]shared2.TransformerInitializer)
transformerInitializerMap[constants.BiteLabel] = transformers.BiteTransformerInitializer initializerMap[constants.BiteLabel] = transformers.BiteTransformer.NewTransformer
transformerInitializerMap[constants.CatFileChopLumpLabel] = transformers.CatFileChopLumpTransformerInitializer initializerMap[constants.CatFileChopLumpLabel] = transformers.CatFileChopLumpTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.CatFileFlipLabel] = transformers.CatFileFlipTransformerInitializer initializerMap[constants.CatFileFlipLabel] = transformers.CatFileFlipTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.CatFilePitVowLabel] = transformers.CatFilePitVowTransformerInitializer initializerMap[constants.CatFilePitVowLabel] = transformers.CatFilePitVowTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.DealLabel] = transformers.DealTransformerInitializer initializerMap[constants.DealLabel] = transformers.DealTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.DentLabel] = transformers.DentTransformerInitializer initializerMap[constants.DentLabel] = transformers.DentTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.DripDripLabel] = transformers.DripDripTransformerInitializer initializerMap[constants.DripDripLabel] = transformers.DripDripTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.DripFileIlkLabel] = transformers.DripFileIlkTransformerInitializer initializerMap[constants.DripFileIlkLabel] = transformers.DripFileIlkTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.DripFileRepoLabel] = transformers.DripFileRepoTransformerInitializer initializerMap[constants.DripFileRepoLabel] = transformers.DripFileRepoTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.DripFileVowLabel] = transformers.DripFileVowTransfromerInitializer initializerMap[constants.DripFileVowLabel] = transformers.DripFileVowTransfromer.NewLogNoteTransformer
transformerInitializerMap[constants.FlapKickLabel] = transformers.FlapKickTransformerInitializer initializerMap[constants.FlapKickLabel] = transformers.FlapKickTransformer.NewTransformer
transformerInitializerMap[constants.FlipKickLabel] = transformers.FlipKickTransformerInitializer initializerMap[constants.FlipKickLabel] = transformers.FlipKickTransformer.NewTransformer
transformerInitializerMap[constants.FlopKickLabel] = transformers.FlopKickTransformerInitializer initializerMap[constants.FlopKickLabel] = transformers.FlopKickTransformer.NewTransformer
transformerInitializerMap[constants.FrobLabel] = transformers.FrobTransformerInitializer initializerMap[constants.FrobLabel] = transformers.FrobTransformer.NewTransformer
transformerInitializerMap[constants.PitFileDebtCeilingLabel] = transformers.PitFileDebtCeilingTransformerInitializer initializerMap[constants.PitFileDebtCeilingLabel] = transformers.PitFileDebtCeilingTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.PitFileIlkLabel] = transformers.PitFileIlkTransformerInitializer initializerMap[constants.PitFileIlkLabel] = transformers.PitFileIlkTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.PriceFeedLabel] = transformers.PriceFeedTransformerInitializer initializerMap[constants.PriceFeedLabel] = transformers.PriceFeedTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.TendLabel] = transformers.TendTransformerInitializer initializerMap[constants.TendLabel] = transformers.TendTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.VatFluxLabel] = transformers.VatFluxTransformerInitializer initializerMap[constants.VatFluxLabel] = transformers.VatFluxTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.VatFoldLabel] = transformers.VatFoldTransformerInitializer initializerMap[constants.VatFoldLabel] = transformers.VatFoldTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.VatGrabLabel] = transformers.VatGrabTransformerInitializer initializerMap[constants.VatGrabLabel] = transformers.VatGrabTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.VatHealLabel] = transformers.VatHealTransformerInitializer initializerMap[constants.VatHealLabel] = transformers.VatHealTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.VatInitLabel] = transformers.VatInitTransformerInitializer initializerMap[constants.VatInitLabel] = transformers.VatInitTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.VatMoveLabel] = transformers.VatMoveTransformerInitializer initializerMap[constants.VatMoveLabel] = transformers.VatMoveTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.VatSlipLabel] = transformers.VatSlipTransformerInitializer initializerMap[constants.VatSlipLabel] = transformers.VatSlipTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.VatTollLabel] = transformers.VatTollTransformerInitializer initializerMap[constants.VatTollLabel] = transformers.VatTollTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.VatTuneLabel] = transformers.VatTuneTransformerInitializer initializerMap[constants.VatTuneLabel] = transformers.VatTuneTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.VowFlogLabel] = transformers.FlogTransformerInitializer initializerMap[constants.VowFlogLabel] = transformers.FlogTransformer.NewLogNoteTransformer
return transformerInitializerMap return initializerMap
} }
func init() { func init() {

71
cmd/erc20.go
View File

@ -1,71 +0,0 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cmd
import (
"time"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block"
"github.com/vulcanize/vulcanizedb/libraries/shared"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
// erc20Cmd represents the erc20 command
var erc20Cmd = &cobra.Command{
Use: "erc20",
Short: "Fetches and persists token supply",
Long: `Fetches the totalSupply for the configured token from each block and persists it in Vulcanize DB.
vulcanizedb erc20 --config environments/public
Expects an ethereum node to be running and requires a .toml config file:
[database]
name = "vulcanize_public"
hostname = "localhost"
port = 5432
[client]
ipcPath = "/Users/user/Library/Ethereum/geth.ipc"
`,
Run: func(cmd *cobra.Command, args []string) {
watchERC20s()
},
}
func watchERC20s() {
ticker := time.NewTicker(5 * time.Second)
defer ticker.Stop()
blockChain := getBlockChain()
db, err := postgres.NewDB(databaseConfig, blockChain.Node())
if err != nil {
log.Fatal("Failed to initialize database.")
}
watcher := shared.Watcher{
DB: *db,
Blockchain: blockChain,
}
watcher.AddTransformers(every_block.TransformerInitializers())
for range ticker.C {
watcher.Execute()
}
}
func init() {
rootCmd.AddCommand(erc20Cmd)
}

21
examples/constants/constants.go
View File

File diff suppressed because one or more lines are too long

27
examples/erc20_test_helpers/database.go
View File

@ -1,27 +0,0 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package erc20_test_helpers
type TokenSupplyDBRow struct {
ID int64
Supply int64
BlockID int64 `db:"block_id"`
TokenAddress string `db:"token_address"`
}
type TransferDBRow struct {
ID int64 `db:"id"`
VulcanizeLogID int64 `db:"vulcanize_log_id"`
}

106
examples/erc20_test_helpers/mocks/mocks.go
View File

@ -1,106 +0,0 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package mocks
import (
"math/big"
"github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
)
type Fetcher struct {
ContractAddress string
Abi string
FetchedBlocks []int64
BlockChain core.BlockChain
supply big.Int
}
func (f *Fetcher) SetSupply(supply string) {
f.supply.SetString(supply, 10)
}
func (f Fetcher) GetBlockChain() core.BlockChain {
return f.BlockChain
}
func (f *Fetcher) FetchSupplyOf(contractAbi string, contractAddress string, blockNumber int64) (big.Int, error) {
f.Abi = contractAbi
f.ContractAddress = contractAddress
f.FetchedBlocks = append(f.FetchedBlocks, blockNumber)
accumulator := big.NewInt(1)
f.supply.Add(&f.supply, accumulator)
return f.supply, nil
}
type TotalSupplyRepository struct {
TotalSuppliesCreated []every_block.TokenSupply
MissingBlockNumbers []int64
StartingBlock int64
EndingBlock int64
}
func (fr *TotalSupplyRepository) Create(supply every_block.TokenSupply) error {
fr.TotalSuppliesCreated = append(fr.TotalSuppliesCreated, supply)
return nil
}
func (fr *TotalSupplyRepository) MissingBlocks(startingBlock int64, highestBlock int64) ([]int64, error) {
fr.StartingBlock = startingBlock
fr.EndingBlock = highestBlock
return fr.MissingBlockNumbers, nil
}
func (fr *TotalSupplyRepository) SetMissingBlocks(missingBlocks []int64) {
fr.MissingBlockNumbers = missingBlocks
}
type FailureRepository struct {
createFail bool
missingBlocksFail bool
missingBlocksNumbers []int64
}
func (fr *FailureRepository) Create(supply every_block.TokenSupply) error {
if fr.createFail {
return fakes.FakeError
} else {
return nil
}
}
func (fr *FailureRepository) MissingBlocks(startingBlock int64, highestBlock int64) ([]int64, error) {
if fr.missingBlocksFail {
return []int64{}, fakes.FakeError
} else {
return fr.missingBlocksNumbers, nil
}
}
func (fr *FailureRepository) SetCreateFail(fail bool) {
fr.createFail = fail
}
func (fr *FailureRepository) SetMissingBlocksFail(fail bool) {
fr.missingBlocksFail = fail
}
func (fr *FailureRepository) SetMissingBlocks(missingBlocks []int64) {
fr.missingBlocksNumbers = missingBlocks
}

19
examples/erc20_watcher/ERC20WatcherREADME.md
View File

@ -1,19 +0,0 @@
# ERC20 Transformers
## Description
The Transformers in this directory are associated with contract functions and events that conform to the [ERC20 Token interface](https://theethereum.wiki/w/index.php/ERC20_Token_Standard#The_ERC20_Token_Standard_Interface).
See `libraries/shared/TransformerREADME.md` for further information regarding the Transformer interface.
## Configuration
In addition to environment configuration mentioned in the main VulcanizeDB README, the ERC20 transformers also need to be configured with contract information for the desired token(s) to be watched. This configuration file is located at `./vulcanizedb/examples/erc20_watcher/config.go`.
## ERC20 Functions
The `everyblock` directory contains transformers that fetch data from the contract itself, via one of the standard functions.
Currently, the `totalSupply` function transformer has been implemented. This transformer will fetch the total supply for the given contract address and persist `total_supply` records in the database.
## Running the transformers
1. If running against a local node, make sure that the node has been started.
1. In a separate terminal run the following command:
`./vulcanizedb erc20 --config <config.toml>`

33
examples/erc20_watcher/config.go
View File

@ -1,33 +0,0 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package erc20_watcher
import "github.com/vulcanize/vulcanizedb/examples/constants"
type ContractConfig struct {
Address string
Abi string
FirstBlock int64
LastBlock int64
Name string
}
var DaiConfig = ContractConfig{
Address: constants.DaiContractAddress,
Abi: constants.DaiAbiString,
FirstBlock: int64(4752008),
LastBlock: -1,
Name: "Dai",
}

33
examples/erc20_watcher/every_block/every_block_suite_test.go
View File

@ -1,33 +0,0 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package every_block_test
import (
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
log "github.com/sirupsen/logrus"
"io/ioutil"
)
func TestEveryBlock(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "EveryBlock Suite")
}
var _ = BeforeSuite(func() {
log.SetOutput(ioutil.Discard)
})

71
examples/erc20_watcher/every_block/fetcher.go
View File

@ -1,71 +0,0 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package every_block
import (
"fmt"
log "github.com/sirupsen/logrus"
"math/big"
"github.com/vulcanize/vulcanizedb/pkg/core"
)
type ERC20FetcherInterface interface {
FetchSupplyOf(contractAbi string, contractAddress string, blockNumber int64) (big.Int, error)
GetBlockChain() core.BlockChain
}
func NewFetcher(blockchain core.BlockChain) Fetcher {
return Fetcher{
Blockchain: blockchain,
}
}
type Fetcher struct {
Blockchain core.BlockChain
ContractAbi string
ContractAddress string
}
type fetcherError struct {
err string
fetchMethod string
}
func (fe *fetcherError) Error() string {
return fmt.Sprintf("Error fetching %s: %s", fe.fetchMethod, fe.err)
}
func newFetcherError(err error, fetchMethod string) *fetcherError {
e := fetcherError{err.Error(), fetchMethod}
log.Info(e.Error())
return &e
}
func (f Fetcher) FetchSupplyOf(contractAbi string, contractAddress string, blockNumber int64) (big.Int, error) {
method := "totalSupply"
var result = new(big.Int)
err := f.Blockchain.FetchContractData(contractAbi, contractAddress, method, nil, &result, blockNumber)
if err != nil {
return *result, newFetcherError(err, method)
}
return *result, nil
}
func (f Fetcher) GetBlockChain() core.BlockChain {
return f.Blockchain
}

82
examples/erc20_watcher/every_block/fetcher_test.go
View File

@ -1,82 +0,0 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package every_block_test
import (
"math/big"
"github.com/ethereum/go-ethereum/ethclient"
"github.com/ethereum/go-ethereum/rpc"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/examples/constants"
"github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/geth"
"github.com/vulcanize/vulcanizedb/pkg/geth/client"
rpc2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
"github.com/vulcanize/vulcanizedb/pkg/geth/node"
)
var _ = Describe("ERC20 Fetcher", func() {
blockNumber := int64(5502914)
Describe("FetchSupplyOf", func() {
It("fetches data from the blockchain with the correct arguments", func() {
fakeBlockchain := fakes.NewMockBlockChain()
testFetcher := every_block.NewFetcher(fakeBlockchain)
testAbi := "testAbi"
testContractAddress := "testContractAddress"
_, err := testFetcher.FetchSupplyOf(testAbi, testContractAddress, blockNumber)
Expect(err).NotTo(HaveOccurred())
expectedResult := big.Int{}
expected := &expectedResult
fakeBlockchain.AssertFetchContractDataCalledWith(testAbi, testContractAddress, "totalSupply", nil, &expected, blockNumber)
})
It("fetches a token's total supply at the given block height", func() {
infuraIPC := "https://mainnet.infura.io/J5Vd2fRtGsw0zZ0Ov3BL"
rawRpcClient, err := rpc.Dial(infuraIPC)
Expect(err).NotTo(HaveOccurred())
rpcClient := client.NewRpcClient(rawRpcClient, infuraIPC)
ethClient := ethclient.NewClient(rawRpcClient)
blockChainClient := client.NewEthClient(ethClient)
node := node.MakeNode(rpcClient)
transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
realFetcher := every_block.NewFetcher(blockChain)
result, err := realFetcher.FetchSupplyOf(constants.DaiAbiString, constants.DaiContractAddress, blockNumber)
Expect(err).NotTo(HaveOccurred())
expectedResult := big.Int{}
expectedResult.SetString("27647235749155415536952630", 10)
Expect(result).To(Equal(expectedResult))
})
It("returns an error if the call to the blockchain fails", func() {
blockChain := fakes.NewMockBlockChain()
blockChain.SetFetchContractDataErr(fakes.FakeError)
errorFetcher := every_block.NewFetcher(blockChain)
result, err := errorFetcher.FetchSupplyOf("", "", 0)
Expect(result.String()).To(Equal("0"))
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("totalSupply"))
Expect(err.Error()).To(ContainSubstring(fakes.FakeError.Error()))
})
})
})

83
examples/erc20_watcher/every_block/integration_test.go
View File

@ -1,83 +0,0 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package every_block_test
import (
"math/big"
"strconv"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/examples/constants"
"github.com/vulcanize/vulcanizedb/examples/erc20_test_helpers"
"github.com/vulcanize/vulcanizedb/examples/erc20_watcher"
"github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/test_config"
)
func setLastBlockOnChain(blockChain *fakes.MockBlockChain, blockNumber int64) {
blockNumberString := strconv.FormatInt(blockNumber, 10)
lastBlockOnChain := big.Int{}
lastBlockOnChain.SetString(blockNumberString, 10)
blockChain.SetLastBlock(&lastBlockOnChain)
}
var _ = Describe("Everyblock transformers", func() {
var db *postgres.DB
var blockChain *fakes.MockBlockChain
var blockNumber int64
var blockId int64
var err error
BeforeEach(func() {
blockChain = fakes.NewMockBlockChain()
blockNumber = erc20_watcher.DaiConfig.FirstBlock
lastBlockNumber := blockNumber + 1
node := test_config.NewTestNode()
db = test_config.NewTestDB(node)
test_config.CleanTestDB(db)
setLastBlockOnChain(blockChain, lastBlockNumber)
blockRepository := repositories.NewBlockRepository(db)
blockId, err = blockRepository.CreateOrUpdateBlock(core.Block{Number: blockNumber})
Expect(err).NotTo(HaveOccurred())
_, err = blockRepository.CreateOrUpdateBlock(core.Block{Number: lastBlockNumber})
Expect(err).NotTo(HaveOccurred())
})
It("creates a token_supply record for each block in the given range", func() {
initializer := every_block.TokenSupplyTransformerInitializer{Config: erc20_watcher.DaiConfig}
transformer := initializer.NewTokenSupplyTransformer(db, blockChain)
transformer.Execute()
var tokenSupplyCount int
err := db.QueryRow(`SELECT COUNT(*) FROM token_supply where block_id = $1`, blockId).Scan(&tokenSupplyCount)
Expect(err).ToNot(HaveOccurred())
Expect(tokenSupplyCount).To(Equal(1))
var tokenSupply erc20_test_helpers.TokenSupplyDBRow
err = db.Get(&tokenSupply, `SELECT * from token_supply where block_id = $1`, blockId)
Expect(err).ToNot(HaveOccurred())
Expect(tokenSupply.BlockID).To(Equal(blockId))
Expect(tokenSupply.TokenAddress).To(Equal(constants.DaiContractAddress))
Expect(tokenSupply.Supply).To(Equal(int64(0)))
})
})

21
examples/erc20_watcher/every_block/model.go
View File

@ -1,21 +0,0 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package every_block
type TokenSupply struct {
Value string
TokenAddress string
BlockNumber int64
}

91
examples/erc20_watcher/every_block/repository.go
View File

@ -1,91 +0,0 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package every_block
import (
"fmt"
log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
type ERC20RepositoryInterface interface {
Create(supply TokenSupply) error
MissingBlocks(startingBlock int64, highestBlock int64) ([]int64, error)
}
type TokenSupplyRepository struct {
*postgres.DB
}
type repositoryError struct {
err string
msg string
blockNumber int64
}
func (re *repositoryError) Error() string {
return fmt.Sprintf(re.msg, re.blockNumber, re.err)
}
func newRepositoryError(err error, msg string, blockNumber int64) error {
e := repositoryError{err.Error(), msg, blockNumber}
log.Println(e.Error())
return &e
}
const (
GetBlockError = "Error fetching block number %d: %s"
InsertTokenSupplyError = "Error inserting token_supply for block number %d: %s"
MissingBlockError = "Error finding missing token_supply records starting at block %d: %s"
)
func (tsp *TokenSupplyRepository) Create(supply TokenSupply) error {
var blockId int
err := tsp.DB.Get(&blockId, `SELECT id FROM blocks WHERE number = $1 AND eth_node_id = $2`, supply.BlockNumber, tsp.NodeID)
if err != nil {
return newRepositoryError(err, GetBlockError, supply.BlockNumber)
}
_, err = tsp.DB.Exec(
`INSERT INTO token_supply (supply, token_address, block_id)
VALUES($1, $2, $3)`,
supply.Value, supply.TokenAddress, blockId)
if err != nil {
return newRepositoryError(err, InsertTokenSupplyError, supply.BlockNumber)
}
return nil
}
func (tsp *TokenSupplyRepository) MissingBlocks(startingBlock int64, highestBlock int64) ([]int64, error) {
blockNumbers := make([]int64, 0)
err := tsp.DB.Select(
&blockNumbers,
`SELECT number FROM BLOCKS
LEFT JOIN token_supply ON blocks.id = block_id
WHERE block_id ISNULL
AND eth_node_id = $1
AND number >= $2
AND number <= $3
LIMIT 20`,
tsp.NodeID,
startingBlock,
highestBlock,
)
if err != nil {
return []int64{}, newRepositoryError(err, MissingBlockError, startingBlock)
}
return blockNumbers, err
}

207
examples/erc20_watcher/every_block/repository_test.go
View File

@ -1,207 +0,0 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package every_block_test
import (
"math/rand"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/examples/erc20_test_helpers"
"github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/test_config"
)
var _ = Describe("ERC20 Token Repository", func() {
var db *postgres.DB
var blockId int64
var blockNumber int64
var repository every_block.TokenSupplyRepository
var blockRepository repositories.BlockRepository
testAddress := "abc"
BeforeEach(func() {
node := test_config.NewTestNode()
db = test_config.NewTestDB(node)
test_config.CleanTestDB(db)
repository = every_block.TokenSupplyRepository{DB: db}
blockRepository = *repositories.NewBlockRepository(db)
blockNumber = rand.Int63()
blockId = test_config.NewTestBlock(blockNumber, blockRepository)
})
Describe("Create", func() {
It("creates a token supply record", func() {
supply := supplyModel(blockNumber, testAddress, "100")
err := repository.Create(supply)
Expect(err).NotTo(HaveOccurred())
dbResult := erc20_test_helpers.TokenSupplyDBRow{}
expectedTokenSupply := erc20_test_helpers.TokenSupplyDBRow{
Supply: int64(100),
BlockID: blockId,
TokenAddress: testAddress,
}
var count int
err = repository.DB.QueryRowx(`SELECT count(*) FROM token_supply`).Scan(&count)
Expect(err).NotTo(HaveOccurred())
Expect(count).To(Equal(1))
err = repository.DB.QueryRowx(`SELECT * FROM token_supply`).StructScan(&dbResult)
Expect(err).NotTo(HaveOccurred())
Expect(dbResult.Supply).To(Equal(expectedTokenSupply.Supply))
Expect(dbResult.BlockID).To(Equal(expectedTokenSupply.BlockID))
Expect(dbResult.TokenAddress).To(Equal(expectedTokenSupply.TokenAddress))
})
It("returns an error if fetching the block's id from the database fails", func() {
errorSupply := supplyModel(-1, "", "")
err := repository.Create(errorSupply)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("sql"))
Expect(err.Error()).To(ContainSubstring("block number -1"))
})
It("returns an error if inserting the token_supply fails", func() {
errorSupply := supplyModel(blockNumber, "", "")
err := repository.Create(errorSupply)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("pq"))
Expect(err.Error()).To(ContainSubstring("token_supply for block number"))
})
})
Describe("When there are multiple nodes", func() {
var node2DB *postgres.DB
var node2BlockRepo *repositories.BlockRepository
var node2BlockId int64
var node2TokenSupplyRepo every_block.TokenSupplyRepository
var tokenSupply every_block.TokenSupply
BeforeEach(func() {
node2DB = createDbForAnotherNode()
//create another block with the same number on node2
node2BlockRepo = repositories.NewBlockRepository(node2DB)
node2BlockId = test_config.NewTestBlock(blockNumber, *node2BlockRepo)
tokenSupply = supplyModel(blockNumber, "abc", "100")
node2TokenSupplyRepo = every_block.TokenSupplyRepository{DB: node2DB}
})
It("only creates token_supply records for the current node (node2)", func() {
err := node2TokenSupplyRepo.Create(tokenSupply)
Expect(err).NotTo(HaveOccurred())
var tokenSupplies []erc20_test_helpers.TokenSupplyDBRow
err = node2TokenSupplyRepo.DB.Select(&tokenSupplies, `SELECT * FROM token_supply`)
Expect(err).NotTo(HaveOccurred())
Expect(len(tokenSupplies)).To(Equal(1))
Expect(tokenSupplies[0].BlockID).To(Equal(node2BlockId))
})
It("only includes missing block numbers for the current node", func() {
//create token_supply on original node
err := repository.Create(tokenSupply)
Expect(err).NotTo(HaveOccurred())
originalNodeMissingBlocks, err := repository.MissingBlocks(blockNumber, blockNumber)
Expect(err).NotTo(HaveOccurred())
Expect(len(originalNodeMissingBlocks)).To(Equal(0))
node2MissingBlocks, err := node2TokenSupplyRepo.MissingBlocks(blockNumber, blockNumber)
Expect(err).NotTo(HaveOccurred())
Expect(len(node2MissingBlocks)).To(Equal(1))
})
})
Describe("MissingBlocks", func() {
It("returns the block numbers for which an associated TokenSupply record hasn't been created", func() {
createTokenSupplyFor(repository, blockNumber)
newBlockNumber := blockNumber + 1
test_config.NewTestBlock(newBlockNumber, blockRepository)
blocks, err := repository.MissingBlocks(blockNumber, newBlockNumber)
Expect(blocks).To(ConsistOf(newBlockNumber))
Expect(err).NotTo(HaveOccurred())
})
It("only returns blocks within the given range", func() {
newBlockNumber := blockNumber + 1
test_config.NewTestBlock(newBlockNumber, blockRepository)
blocks, err := repository.MissingBlocks(blockNumber, blockNumber)
Expect(blocks).NotTo(ConsistOf(newBlockNumber))
Expect(err).NotTo(HaveOccurred())
})
It("does not return numbers that already have an associated TokenSupply record", func() {
createTokenSupplyFor(repository, blockNumber)
blocks, err := repository.MissingBlocks(blockNumber, blockNumber)
Expect(blocks).To(BeEmpty())
Expect(err).NotTo(HaveOccurred())
})
})
It("deletes the token supply record when the associated block is deleted", func() {
err := repository.Create(every_block.TokenSupply{BlockNumber: blockNumber, Value: "0"})
Expect(err).NotTo(HaveOccurred())
var count int
err = repository.DB.QueryRowx(`SELECT count(*) FROM token_supply`).Scan(&count)
Expect(err).NotTo(HaveOccurred())
Expect(count).To(Equal(1))
_, err = db.Query(`DELETE FROM blocks`)
Expect(err).NotTo(HaveOccurred())
err = repository.DB.QueryRowx(`SELECT count(*) FROM token_supply`).Scan(&count)
Expect(err).NotTo(HaveOccurred())
Expect(count).To(Equal(0))
})
})
func supplyModel(blockNumber int64, tokenAddress string, supplyValue string) every_block.TokenSupply {
return every_block.TokenSupply{
Value: supplyValue,
TokenAddress: tokenAddress,
BlockNumber: int64(blockNumber),
}
}
func createTokenSupplyFor(repository every_block.TokenSupplyRepository, blockNumber int64) {
err := repository.Create(every_block.TokenSupply{BlockNumber: blockNumber, Value: "0"})
Expect(err).NotTo(HaveOccurred())
}
func createDbForAnotherNode() *postgres.DB {
anotherNode := core.Node{
GenesisBlock: "GENESIS",
NetworkID: 1,
ID: "testNodeId",
ClientName: "Geth/v1.7.2-stable-1db4ecdc/darwin-amd64/go1.9",
}
return test_config.NewTestDB(anotherNode)
}

118
examples/erc20_watcher/every_block/transformer.go
View File

@ -1,118 +0,0 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package every_block
import (
"fmt"
"math/big"
log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/examples/erc20_watcher"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
)
type Transformer struct {
Fetcher ERC20FetcherInterface
Repository ERC20RepositoryInterface
Config erc20_watcher.ContractConfig
}
func (t *Transformer) SetConfiguration(config erc20_watcher.ContractConfig) {
t.Config = config
}
type TokenSupplyTransformerInitializer struct {
Config erc20_watcher.ContractConfig
}
func (i TokenSupplyTransformerInitializer) NewTokenSupplyTransformer(db *postgres.DB, blockchain core.BlockChain) shared.Transformer {
fetcher := NewFetcher(blockchain)
repository := TokenSupplyRepository{DB: db}
transformer := Transformer{
Fetcher: &fetcher,
Repository: &repository,
Config: i.Config,
}
return transformer
}
const (
FetchingBlocksError = "Error fetching missing blocks starting at block number %d: %s"
FetchingSupplyError = "Error fetching supply for block %d: %s"
CreateSupplyError = "Error inserting token_supply for block %d: %s"
)
type transformerError struct {
err string
blockNumber int64
msg string
}
func (te *transformerError) Error() string {
return fmt.Sprintf(te.msg, te.blockNumber, te.err)
}
func newTransformerError(err error, blockNumber int64, msg string) error {
e := transformerError{err.Error(), blockNumber, msg}
log.Println(e.Error())
return &e
}
func (t Transformer) Execute() error {
var upperBoundBlock int64
blockchain := t.Fetcher.GetBlockChain()
lastBlock := blockchain.LastBlock().Int64()
if t.Config.LastBlock == -1 {
upperBoundBlock = lastBlock
} else {
upperBoundBlock = t.Config.LastBlock
}
blocks, err := t.Repository.MissingBlocks(t.Config.FirstBlock, upperBoundBlock)
if err != nil {
return newTransformerError(err, t.Config.FirstBlock, FetchingBlocksError)
}
log.Printf("Fetching totalSupply for %d blocks", len(blocks))
for _, blockNumber := range blocks {
totalSupply, err := t.Fetcher.FetchSupplyOf(t.Config.Abi, t.Config.Address, blockNumber)
if err != nil {
return newTransformerError(err, blockNumber, FetchingSupplyError)
}
model := createTokenSupplyModel(totalSupply, t.Config.Address, blockNumber)
err = t.Repository.Create(model)
if err != nil {
return newTransformerError(err, blockNumber, CreateSupplyError)
}
}
return nil
}
func createTokenSupplyModel(totalSupply big.Int, address string, blockNumber int64) TokenSupply {
return TokenSupply{
Value: totalSupply.String(),
TokenAddress: address,
BlockNumber: blockNumber,
}
}

188
examples/erc20_watcher/every_block/transformer_test.go
View File

@ -1,188 +0,0 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package every_block_test
import (
"math/big"
"math/rand"
"strconv"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/examples/constants"
"github.com/vulcanize/vulcanizedb/examples/erc20_test_helpers/mocks"
"github.com/vulcanize/vulcanizedb/examples/erc20_watcher"
"github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
)
var testContractConfig = erc20_watcher.ContractConfig{
Address: constants.DaiContractAddress,
Abi: constants.DaiAbiString,
FirstBlock: int64(4752008),
LastBlock: int64(5750050),
Name: "Dai",
}
var config = testContractConfig
var _ = Describe("Everyblock transformer", func() {
var fetcher mocks.Fetcher
var repository mocks.TotalSupplyRepository
var transformer every_block.Transformer
var blockChain *fakes.MockBlockChain
var initialSupply = "27647235749155415536952630"
var initialSupplyPlusOne = "27647235749155415536952631"
var initialSupplyPlusTwo = "27647235749155415536952632"
var initialSupplyPlusThree = "27647235749155415536952633"
var defaultLastBlock = big.Int{}
BeforeEach(func() {
blockChain = fakes.NewMockBlockChain()
blockChain.SetLastBlock(&defaultLastBlock)
fetcher = mocks.Fetcher{BlockChain: blockChain}
fetcher.SetSupply(initialSupply)
repository = mocks.TotalSupplyRepository{}
repository.SetMissingBlocks([]int64{config.FirstBlock})
//setting the mock repository to return the first block as the missing blocks
transformer = every_block.Transformer{
Fetcher: &fetcher,
Repository: &repository,
}
transformer.SetConfiguration(config)
})
It("fetches and persists the total supply of a token for a single block", func() {
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(len(fetcher.FetchedBlocks)).To(Equal(1))
Expect(fetcher.FetchedBlocks).To(ConsistOf(config.FirstBlock))
Expect(fetcher.Abi).To(Equal(config.Abi))
Expect(fetcher.ContractAddress).To(Equal(config.Address))
Expect(repository.StartingBlock).To(Equal(config.FirstBlock))
Expect(repository.EndingBlock).To(Equal(config.LastBlock))
Expect(len(repository.TotalSuppliesCreated)).To(Equal(1))
expectedSupply := big.Int{}
expectedSupply.SetString(initialSupply, 10)
expectedSupply.Add(&expectedSupply, big.NewInt(1))
Expect(repository.TotalSuppliesCreated[0].Value).To(Equal(expectedSupply.String()))
})
It("retrieves the total supply for every missing block", func() {
missingBlocks := []int64{
config.FirstBlock,
config.FirstBlock + 1,
config.FirstBlock + 2,
}
repository.SetMissingBlocks(missingBlocks)
transformer.Execute()
Expect(len(fetcher.FetchedBlocks)).To(Equal(3))
Expect(fetcher.FetchedBlocks).To(ConsistOf(config.FirstBlock, config.FirstBlock+1, config.FirstBlock+2))
Expect(fetcher.Abi).To(Equal(config.Abi))
Expect(fetcher.ContractAddress).To(Equal(config.Address))
Expect(len(repository.TotalSuppliesCreated)).To(Equal(3))
Expect(repository.TotalSuppliesCreated[0].Value).To(Equal(initialSupplyPlusOne))
Expect(repository.TotalSuppliesCreated[1].Value).To(Equal(initialSupplyPlusTwo))
Expect(repository.TotalSuppliesCreated[2].Value).To(Equal(initialSupplyPlusThree))
})
It("uses the set contract configuration", func() {
repository.SetMissingBlocks([]int64{testContractConfig.FirstBlock})
transformer.SetConfiguration(testContractConfig)
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(fetcher.FetchedBlocks).To(ConsistOf(testContractConfig.FirstBlock))
Expect(fetcher.Abi).To(Equal(testContractConfig.Abi))
Expect(fetcher.ContractAddress).To(Equal(testContractConfig.Address))
Expect(repository.StartingBlock).To(Equal(testContractConfig.FirstBlock))
Expect(repository.EndingBlock).To(Equal(testContractConfig.LastBlock))
Expect(len(repository.TotalSuppliesCreated)).To(Equal(1))
expectedTokenSupply := every_block.TokenSupply{
Value: initialSupplyPlusOne,
TokenAddress: testContractConfig.Address,
BlockNumber: testContractConfig.FirstBlock,
}
Expect(repository.TotalSuppliesCreated[0]).To(Equal(expectedTokenSupply))
})
It("uses the most recent block if the Config.LastBlock is -1", func() {
testContractConfig.LastBlock = -1
transformer.SetConfiguration(testContractConfig)
randomBlockNumber := rand.Int63()
numberToString := strconv.FormatInt(randomBlockNumber, 10)
mostRecentBlock := big.Int{}
mostRecentBlock.SetString(numberToString, 10)
blockChain.SetLastBlock(&mostRecentBlock)
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(repository.EndingBlock).To(Equal(randomBlockNumber))
})
It("returns an error if the call to get missing blocks fails", func() {
failureRepository := mocks.FailureRepository{}
failureRepository.SetMissingBlocksFail(true)
transformer = every_block.Transformer{
Fetcher: &fetcher,
Repository: &failureRepository,
}
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring(fakes.FakeError.Error()))
Expect(err.Error()).To(ContainSubstring("fetching missing blocks"))
})
It("returns an error if the call to the blockChain fails", func() {
failureBlockchain := fakes.NewMockBlockChain()
failureBlockchain.SetLastBlock(&defaultLastBlock)
failureBlockchain.SetFetchContractDataErr(fakes.FakeError)
fetcher := every_block.NewFetcher(failureBlockchain)
transformer = every_block.Transformer{
Fetcher: &fetcher,
Repository: &repository,
}
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring(fakes.FakeError.Error()))
Expect(err.Error()).To(ContainSubstring("supply"))
})
It("returns an error if the call to save the token_supply fails", func() {
failureRepository := mocks.FailureRepository{}
failureRepository.SetMissingBlocks([]int64{config.FirstBlock})
failureRepository.SetCreateFail(true)
transformer = every_block.Transformer{
Fetcher: &fetcher,
Repository: &failureRepository,
}
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring(fakes.FakeError.Error()))
Expect(err.Error()).To(ContainSubstring("supply"))
})
})

28
examples/erc20_watcher/every_block/transformers.go
View File

@ -1,28 +0,0 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package every_block
import (
"github.com/vulcanize/vulcanizedb/examples/erc20_watcher"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
)
func TransformerInitializers() []shared.TransformerInitializer {
config := erc20_watcher.DaiConfig
initializer := TokenSupplyTransformerInitializer{config}
return []shared.TransformerInitializer{
initializer.NewTokenSupplyTransformer,
}
}

81
libraries/shared/watcher.go
View File

@ -1,6 +1,8 @@
package shared package shared
import ( import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/log"
"github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
@ -8,21 +10,82 @@ import (
type Watcher struct { type Watcher struct {
Transformers []shared.Transformer Transformers []shared.Transformer
DB postgres.DB DB *postgres.DB
Blockchain core.BlockChain Fetcher shared.LogFetcher
Chunker shared.Chunker
Addresses []common.Address
Topics []common.Hash
} }
func (watcher *Watcher) AddTransformers(us []shared.TransformerInitializer) { func NewWatcher(db *postgres.DB, bc core.BlockChain) Watcher {
for _, transformerInitializer := range us { chunker := shared.NewLogChunker()
transformer := transformerInitializer(&watcher.DB, watcher.Blockchain) fetcher := shared.NewFetcher(bc)
watcher.Transformers = append(watcher.Transformers, transformer) return Watcher{
DB: db,
Fetcher: fetcher,
Chunker: chunker,
} }
} }
// Adds transformers to the watcher and updates the chunker, so that it will consider the new transformers.
func (watcher *Watcher) AddTransformers(initializers []shared.TransformerInitializer) {
var contractAddresses []common.Address
var topic0s []common.Hash
var configs []shared.TransformerConfig
for _, initializer := range initializers {
transformer := initializer(watcher.DB)
watcher.Transformers = append(watcher.Transformers, transformer)
config := transformer.GetConfig()
configs = append(configs, config)
addresses := shared.HexStringsToAddresses(config.ContractAddresses)
contractAddresses = append(contractAddresses, addresses...)
topic0s = append(topic0s, common.HexToHash(config.Topic))
}
watcher.Addresses = append(watcher.Addresses, contractAddresses...)
watcher.Topics = append(watcher.Topics, topic0s...)
watcher.Chunker.AddConfigs(configs)
}
func (watcher *Watcher) Execute() error { func (watcher *Watcher) Execute() error {
var err error checkedColumnNames, err := shared.GetCheckedColumnNames(watcher.DB)
for _, transformer := range watcher.Transformers { if err != nil {
err = transformer.Execute() return err
}
notCheckedSQL := shared.CreateNotCheckedSQL(checkedColumnNames)
// TODO Handle start and end numbers in transformers
missingHeaders, err := shared.MissingHeaders(0, -1, watcher.DB, notCheckedSQL)
if err != nil {
log.Error("Fetching of missing headers failed in watcher!")
return err
}
for _, header := range missingHeaders {
// TODO Extend FetchLogs for doing several blocks at a time
logs, err := watcher.Fetcher.FetchLogs(watcher.Addresses, watcher.Topics, header)
if err != nil {
// TODO Handle fetch error in watcher
log.Error("Error while fetching logs for header %v in watcher", header.Id)
return err
}
chunkedLogs := watcher.Chunker.ChunkLogs(logs)
// Can't quit early and mark as checked if there are no logs. If we are running continuousLogSync,
// not all logs we're interested in might have been fetched.
for _, transformer := range watcher.Transformers {
transformerName := transformer.GetConfig().TransformerName
logChunk := chunkedLogs[transformerName]
err = transformer.Execute(logChunk, header)
if err != nil {
log.Error("%v transformer failed to execute in watcher: %v", transformerName, err)
return err
}
}
} }
return err return err
} }

173
libraries/shared/watcher_test.go
View File

@ -2,69 +2,162 @@ package shared_test
import ( import (
"errors" "errors"
"github.com/ethereum/go-ethereum"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/libraries/shared" "github.com/vulcanize/vulcanizedb/libraries/shared"
"github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
shared2 "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" shared2 "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks"
"github.com/vulcanize/vulcanizedb/test_config"
) )
type MockTransformer struct {
executeWasCalled bool
executeError error
}
func (mh *MockTransformer) Execute() error {
if mh.executeError != nil {
return mh.executeError
}
mh.executeWasCalled = true
return nil
}
func fakeTransformerInitializer(db *postgres.DB, blockchain core.BlockChain) shared2.Transformer {
return &MockTransformer{}
}
var _ = Describe("Watcher", func() { var _ = Describe("Watcher", func() {
It("Adds transformers", func() { It("initialises correctly", func() {
watcher := shared.Watcher{} db := test_config.NewTestDB(core.Node{ID: "testNode"})
bc := fakes.NewMockBlockChain()
watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformerInitializer}) watcher := shared.NewWatcher(db, bc)
Expect(watcher.DB).To(Equal(db))
Expect(watcher.Fetcher).NotTo(BeNil())
Expect(watcher.Chunker).NotTo(BeNil())
})
It("adds transformers", func() {
watcher := shared.NewWatcher(nil, nil)
fakeTransformer := &mocks.MockTransformer{}
fakeTransformer.SetTransformerConfig(mocks.FakeTransformerConfig)
watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformer.FakeTransformerInitializer})
Expect(len(watcher.Transformers)).To(Equal(1)) Expect(len(watcher.Transformers)).To(Equal(1))
Expect(watcher.Transformers).To(ConsistOf(&MockTransformer{})) Expect(watcher.Transformers).To(ConsistOf(fakeTransformer))
Expect(watcher.Topics).To(Equal([]common.Hash{common.HexToHash("FakeTopic")}))
Expect(watcher.Addresses).To(Equal([]common.Address{common.HexToAddress("FakeAddress")}))
}) })
It("Adds transformers from multiple sources", func() { It("adds transformers from multiple sources", func() {
watcher := shared.Watcher{} watcher := shared.NewWatcher(nil, nil)
fakeTransformer1 := &mocks.MockTransformer{}
fakeTransformer1.SetTransformerConfig(mocks.FakeTransformerConfig)
watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformerInitializer}) fakeTransformer2 := &mocks.MockTransformer{}
watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformerInitializer}) fakeTransformer2.SetTransformerConfig(mocks.FakeTransformerConfig)
watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformer1.FakeTransformerInitializer})
watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformer2.FakeTransformerInitializer})
Expect(len(watcher.Transformers)).To(Equal(2)) Expect(len(watcher.Transformers)).To(Equal(2))
Expect(watcher.Topics).To(Equal([]common.Hash{common.HexToHash("FakeTopic"),
common.HexToHash("FakeTopic")}))
Expect(watcher.Addresses).To(Equal([]common.Address{common.HexToAddress("FakeAddress"),
common.HexToAddress("FakeAddress")}))
}) })
It("Executes each transformer", func() { Describe("with missing headers", func() {
watcher := shared.Watcher{} var (
fakeTransformer := &MockTransformer{} db *postgres.DB
watcher.Transformers = []shared2.Transformer{fakeTransformer} watcher shared.Watcher
mockBlockChain fakes.MockBlockChain
fakeTransformer *mocks.MockTransformer
headerRepository repositories.HeaderRepository
repository mocks.MockWatcherRepository
)
watcher.Execute() BeforeEach(func() {
db = test_config.NewTestDB(test_config.NewTestNode())
test_config.CleanTestDB(db)
mockBlockChain = fakes.MockBlockChain{}
headerRepository = repositories.NewHeaderRepository(db)
_, err := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
Expect(err).NotTo(HaveOccurred())
Expect(fakeTransformer.executeWasCalled).To(BeTrue()) repository = mocks.MockWatcherRepository{}
}) watcher = shared.NewWatcher(db, &mockBlockChain)
})
It("Returns an error if transformer returns an error", func() { It("executes each transformer", func() {
watcher := shared.Watcher{} fakeTransformer = &mocks.MockTransformer{}
fakeTransformer := &MockTransformer{executeError: errors.New("Something bad happened")} watcher.Transformers = []shared2.Transformer{fakeTransformer}
watcher.Transformers = []shared2.Transformer{fakeTransformer} repository.SetMissingHeaders([]core.Header{fakes.FakeHeader})
err := watcher.Execute() err := watcher.Execute()
Expect(err).To(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(fakeTransformer.executeWasCalled).To(BeFalse()) Expect(fakeTransformer.ExecuteWasCalled).To(BeTrue())
})
It("returns an error if transformer returns an error", func() {
fakeTransformer = &mocks.MockTransformer{ExecuteError: errors.New("Something bad happened")}
watcher.Transformers = []shared2.Transformer{fakeTransformer}
repository.SetMissingHeaders([]core.Header{fakes.FakeHeader})
err := watcher.Execute()
Expect(err).To(HaveOccurred())
Expect(fakeTransformer.ExecuteWasCalled).To(BeFalse())
})
It("passes only relevant logs to each transformer", func() {
transformerA := &mocks.MockTransformer{}
transformerB := &mocks.MockTransformer{}
configA := shared2.TransformerConfig{TransformerName: "transformerA",
ContractAddresses: []string{"0x000000000000000000000000000000000000000A"},
Topic: "0xA"}
configB := shared2.TransformerConfig{TransformerName: "transformerB",
ContractAddresses: []string{"0x000000000000000000000000000000000000000b"},
Topic: "0xB"}
transformerA.SetTransformerConfig(configA)
transformerB.SetTransformerConfig(configB)
logA := types.Log{Address: common.HexToAddress("0xA"),
Topics: []common.Hash{common.HexToHash("0xA")}}
logB := types.Log{Address: common.HexToAddress("0xB"),
Topics: []common.Hash{common.HexToHash("0xB")}}
mockBlockChain.SetGetEthLogsWithCustomQueryReturnLogs([]types.Log{logA, logB})
repository.SetMissingHeaders([]core.Header{fakes.FakeHeader})
watcher = shared.NewWatcher(db, &mockBlockChain)
watcher.AddTransformers([]shared2.TransformerInitializer{
transformerA.FakeTransformerInitializer, transformerB.FakeTransformerInitializer})
err := watcher.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(transformerA.PassedLogs).To(Equal([]types.Log{logA}))
Expect(transformerB.PassedLogs).To(Equal([]types.Log{logB}))
})
Describe("uses the LogFetcher correctly:", func() {
BeforeEach(func() {
repository.SetMissingHeaders([]core.Header{fakes.FakeHeader})
})
It("fetches logs", func() {
err := watcher.Execute()
Expect(err).NotTo(HaveOccurred())
fakeHash := common.HexToHash(fakes.FakeHeader.Hash)
mockBlockChain.AssertGetEthLogsWithCustomQueryCalledWith(ethereum.FilterQuery{
BlockHash: &fakeHash,
Addresses: nil,
Topics: [][]common.Hash{nil},
})
})
It("propagates log fetcher errors", func() {
fetcherError := errors.New("FetcherError")
mockBlockChain.SetGetEthLogsWithCustomQueryErr(fetcherError)
err := watcher.Execute()
Expect(err).To(MatchError(fetcherError))
})
})
}) })
}) })

1
pkg/datastore/postgres/repositories/header_repository.go
View File

@ -3,7 +3,6 @@ package repositories
import ( import (
"database/sql" "database/sql"
"errors" "errors"
"github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
) )

19
pkg/transformers/DOCUMENTATION.md
View File

@ -25,7 +25,7 @@ The transformer process for each of these different log types is the same, excep
## Creating a Transformer ## Creating a Transformer
1. Pull an example event (from kovan / ganache etc.) 1. Pull an example event (from kovan / ganache etc.)
1. Add event sig to [`constants.go`](./shared/constants.go) 1. Add event & method sig, contract address, `checked_headers` column name, and label to relevant files in [`constants`](./shared/constants)
1. Write a test for the event sig in [`event_signature_generator_test.go`](./shared/event_signature_generator_test.go) 1. Write a test for the event sig in [`event_signature_generator_test.go`](./shared/event_signature_generator_test.go)
1. Create DB table (using [`create_migration`](../../scripts/create_migration)) 1. Create DB table (using [`create_migration`](../../scripts/create_migration))
1. Create columns in `checked_headers` in the _same_ migration 1. Create columns in `checked_headers` in the _same_ migration
@ -36,23 +36,23 @@ The transformer process for each of these different log types is the same, excep
1. Write repository + repository tests 1. Write repository + repository tests
1. Create converter + repository mocks 1. Create converter + repository mocks
1. Create an config object [`shared.TransformerConfig`](./shared/transformer.go) in `config.go` 1. Create an config object [`shared.TransformerConfig`](./shared/transformer.go) in `config.go`
1. Create transformer + transformer tests 1. Wire up transformer in [`transformers.go`](./transformers.go), remembering to add it to `TransformerInitializers()`
1. Wire up transformer in [`transformers.go`](./transformers.go)
1. Wire up transformer in [`continuousLogSync.go`](../../cmd/continuousLogSync.go) 1. Wire up transformer in [`continuousLogSync.go`](../../cmd/continuousLogSync.go)
1. Manually trigger an event and check that it gets persisted to postgres 1. Manually trigger an event and check that it gets persisted to postgres
1. Create an integration test for the shiny new transformer in [`integration_tests`](./integration_tests)
**Fetching Logs** **Fetching Logs**
1. Generate an example raw log event, by either: 1. Generate an example raw log event, by either:
- Pulling the log directly from the Kovan deployment ([constants.go](https://github.com/8thlight/maker-vulcanizedb/blob/master/pkg/transformers/shared/constants.go)). - Pulling the log directly from the Kovan deployment ([address.go](https://github.com/8thlight/maker-vulcanizedb/blob/master/pkg/transformers/shared/constants/address.go)).
- Deploying the contract to a local chain and emiting the event manually. - Deploying the contract to a local chain and emiting the event manually.
1. Fetch the logs from the chain based on the example event's topic zero: 1. Fetch the logs from the chain based on the example event's topic zero:
- The topic zero is based on the keccak-256 hash of the log event's method signature. These are located in [`pkg/transformers/shared/constants.go`](./shared/constants.go). - The topic zero is based on the keccak-256 hash of the log event's method signature. These are located in [`pkg/transformers/shared/constants/signature.go`](./shared/constants/signature.go).
- Most transformers use `shared.LogFetcher` to fetch all logs that match the given topic zero for that log event. - Fetching is done in batch from the [`watcher`](../../libraries/shared/watcher.go).
- Since there are multiple price feed contract address that all use the same `LogValue` event, we have a special implementation of a fetcher specifically for price feeds that can query using all of the contract addresses at once, thus only needing to make one call to the blockchain. - The logs are then chunked up by the [`chunker`](./shared/log_chunker.go) before being delegated to each transformer.
**Coverting logs** **Coverting logs**
@ -105,8 +105,7 @@ The transformer process for each of these different log types is the same, excep
- The `checked_headers` table allows us to keep track of which headers have been checked for a given log type. - The `checked_headers` table allows us to keep track of which headers have been checked for a given log type.
- To create a new migration file: `./scripts/create_migration create_flop_kick` - To create a new migration file: `./scripts/create_migration create_flop_kick`
- See `db/migrations/1536942529_create_flop_kick.up.sql`. - See `db/migrations/1536942529_create_flop_kick.up.sql`.
- The specific log event tables are all created in the `maker` - The specific log event tables are all created in the `maker` schema.
schema.
- There is a one-many association between `headers` and the log - There is a one-many association between `headers` and the log
event tables. This is so that if a header is removed due to a reorg, the associated log event records are also removed. event tables. This is so that if a header is removed due to a reorg, the associated log event records are also removed.
- To run the migrations: `make migrate HOST=local_host PORT=5432 NAME=vulcanize_private` - To run the migrations: `make migrate HOST=local_host PORT=5432 NAME=vulcanize_private`
@ -120,7 +119,7 @@ The transformer process for each of these different log types is the same, excep
**Wire each component up in the transformer** **Wire each component up in the transformer**
- We use a TransformerInitializer struct for each transformer so that we can inject ethRPC and postgresDB connections as well as configuration data (including the contract address, block range, etc.) into the transformer. The TransformerInitializer interface is defined in `pkg/transformers/shared/transformer.go`. - We use a [`TransformerInitializer`](./shared/transformer.go) struct for each transformer so that we can inject ethRPC and postgresDB connections as well as configuration data (including the contract address, block range, etc.) into the transformer.
- See any of `pkg/transformers/flop_kick/transformer.go` - See any of `pkg/transformers/flop_kick/transformer.go`
- All of the transformers are then initialized in `pkg/transformers/transformers.go` with their configuration. - All of the transformers are then initialized in `pkg/transformers/transformers.go` with their configuration.
- The transformers can be executed by using the `continuousLogSync` command, which can be configured to run specific transformers or all transformers. - The transformers can be executed by using the `continuousLogSync` command, which can be configured to run specific transformers or all transformers.

5
pkg/transformers/bite/repository.go
View File

@ -16,7 +16,6 @@ package bite
import ( import (
"fmt" "fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -65,7 +64,3 @@ func (repository BiteRepository) Create(headerID int64, models []interface{}) er
func (repository BiteRepository) MarkHeaderChecked(headerID int64) error { func (repository BiteRepository) MarkHeaderChecked(headerID int64) error {
return shared.MarkHeaderChecked(headerID, repository.db, constants.BiteChecked) return shared.MarkHeaderChecked(headerID, repository.db, constants.BiteChecked)
} }
func (repository BiteRepository) MissingHeaders(startingBlockNumber int64, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.BiteChecked)
}

10
pkg/transformers/bite/repository_test.go
View File

@ -86,14 +86,4 @@ var _ = Describe("Bite repository", func() {
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
}) })
Describe("MissingHeaders", func() {
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
Repository: &biteRepository,
RepositoryTwo: &bite.BiteRepository{},
}
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
})
}) })

5
pkg/transformers/cat_file/chop_lump/repository.go
View File

@ -16,7 +16,6 @@ package chop_lump
import ( import (
"fmt" "fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -62,10 +61,6 @@ func (repository CatFileChopLumpRepository) MarkHeaderChecked(headerID int64) er
return shared.MarkHeaderChecked(headerID, repository.db, constants.CatFileChopLumpChecked) return shared.MarkHeaderChecked(headerID, repository.db, constants.CatFileChopLumpChecked)
} }
func (repository CatFileChopLumpRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.CatFileChopLumpChecked)
}
func (repository *CatFileChopLumpRepository) SetDB(db *postgres.DB) { func (repository *CatFileChopLumpRepository) SetDB(db *postgres.DB) {
repository.db = db repository.db = db
} }

9
pkg/transformers/cat_file/chop_lump/repository_test.go
View File

@ -98,13 +98,4 @@ var _ = Describe("Cat file chop lump repository", func() {
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
}) })
Describe("MissingHeaders", func() {
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
Repository: &catFileRepository,
RepositoryTwo: &chop_lump.CatFileChopLumpRepository{},
}
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
})
}) })

5
pkg/transformers/cat_file/flip/repository.go
View File

@ -16,7 +16,6 @@ package flip
import ( import (
"fmt" "fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -61,10 +60,6 @@ func (repository CatFileFlipRepository) MarkHeaderChecked(headerID int64) error
return shared.MarkHeaderChecked(headerID, repository.db, constants.CatFileFlipChecked) return shared.MarkHeaderChecked(headerID, repository.db, constants.CatFileFlipChecked)
} }
func (repository CatFileFlipRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.CatFileFlipChecked)
}
func (repository *CatFileFlipRepository) SetDB(db *postgres.DB) { func (repository *CatFileFlipRepository) SetDB(db *postgres.DB) {
repository.db = db repository.db = db
} }

8
pkg/transformers/cat_file/flip/repository_test.go
View File

@ -83,12 +83,4 @@ var _ = Describe("Cat file flip repository", func() {
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
}) })
Describe("MissingHeaders", func() {
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
Repository: &catFileFlipRepository,
RepositoryTwo: &flip.CatFileFlipRepository{},
}
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
})
}) })

5
pkg/transformers/cat_file/pit_vow/repository.go
View File

@ -16,7 +16,6 @@ package pit_vow
import ( import (
"fmt" "fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -61,10 +60,6 @@ func (repository CatFilePitVowRepository) MarkHeaderChecked(headerID int64) erro
return shared.MarkHeaderChecked(headerID, repository.db, constants.CatFilePitVowChecked) return shared.MarkHeaderChecked(headerID, repository.db, constants.CatFilePitVowChecked)
} }
func (repository CatFilePitVowRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.CatFilePitVowChecked)
}
func (repository *CatFilePitVowRepository) SetDB(db *postgres.DB) { func (repository *CatFilePitVowRepository) SetDB(db *postgres.DB) {
repository.db = db repository.db = db
} }

9
pkg/transformers/cat_file/pit_vow/repository_test.go
View File

@ -81,13 +81,4 @@ var _ = Describe("Cat file pit vow repository", func() {
} }
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
}) })
Describe("MissingHeaders", func() {
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
Repository: &catFilePitVowRepository,
RepositoryTwo: &pit_vow.CatFilePitVowRepository{},
}
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
})
}) })

5
pkg/transformers/deal/repository.go
View File

@ -16,7 +16,6 @@ package deal
import ( import (
"fmt" "fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -62,10 +61,6 @@ func (repository DealRepository) MarkHeaderChecked(headerID int64) error {
return shared.MarkHeaderChecked(headerID, repository.db, constants.DealChecked) return shared.MarkHeaderChecked(headerID, repository.db, constants.DealChecked)
} }
func (repository DealRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DealChecked)
}
func (repository *DealRepository) SetDB(db *postgres.DB) { func (repository *DealRepository) SetDB(db *postgres.DB) {
repository.db = db repository.db = db
} }

9
pkg/transformers/deal/repository_test.go
View File

@ -84,13 +84,4 @@ var _ = Describe("Deal Repository", func() {
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
}) })
Describe("MissingHeaders", func() {
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
Repository: &dealRepository,
RepositoryTwo: &deal.DealRepository{},
}
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
})
}) })

5
pkg/transformers/dent/repository.go
View File

@ -16,7 +16,6 @@ package dent
import ( import (
"fmt" "fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -67,10 +66,6 @@ func (repository DentRepository) MarkHeaderChecked(headerId int64) error {
return shared.MarkHeaderChecked(headerId, repository.db, constants.DentChecked) return shared.MarkHeaderChecked(headerId, repository.db, constants.DentChecked)
} }
func (repository DentRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DentChecked)
}
func (repository *DentRepository) SetDB(db *postgres.DB) { func (repository *DentRepository) SetDB(db *postgres.DB) {
repository.db = db repository.db = db
} }

9
pkg/transformers/dent/repository_test.go
View File

@ -92,13 +92,4 @@ var _ = Describe("Dent Repository", func() {
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
}) })
Describe("MissingHeaders", func() {
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
Repository: &dentRepository,
RepositoryTwo: &dent.DentRepository{},
}
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
})
}) })

5
pkg/transformers/drip_drip/repository.go
View File

@ -16,7 +16,6 @@ package drip_drip
import ( import (
"fmt" "fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -61,10 +60,6 @@ func (repository DripDripRepository) MarkHeaderChecked(headerID int64) error {
return shared.MarkHeaderChecked(headerID, repository.db, constants.DripDripChecked) return shared.MarkHeaderChecked(headerID, repository.db, constants.DripDripChecked)
} }
func (repository DripDripRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DripDripChecked)
}
func (repository *DripDripRepository) SetDB(db *postgres.DB) { func (repository *DripDripRepository) SetDB(db *postgres.DB) {
repository.db = db repository.db = db
} }

9
pkg/transformers/drip_drip/repository_test.go
View File

@ -80,13 +80,4 @@ var _ = Describe("Drip drip repository", func() {
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
}) })
Describe("MissingHeaders", func() {
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
Repository: &dripDripRepository,
RepositoryTwo: &drip_drip.DripDripRepository{},
}
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
})
}) })

5
pkg/transformers/drip_file/ilk/repository.go
View File

@ -16,7 +16,6 @@ package ilk
import ( import (
"fmt" "fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -64,10 +63,6 @@ func (repository DripFileIlkRepository) MarkHeaderChecked(headerID int64) error
return shared.MarkHeaderChecked(headerID, repository.db, constants.DripFileIlkChecked) return shared.MarkHeaderChecked(headerID, repository.db, constants.DripFileIlkChecked)
} }
func (repository DripFileIlkRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DripFileIlkChecked)
}
func (repository *DripFileIlkRepository) SetDB(db *postgres.DB) { func (repository *DripFileIlkRepository) SetDB(db *postgres.DB) {
repository.db = db repository.db = db
} }

9
pkg/transformers/drip_file/ilk/repository_test.go
View File

@ -82,13 +82,4 @@ var _ = Describe("Drip file ilk repository", func() {
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
}) })
Describe("MissingHeaders", func() {
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
Repository: &dripFileIlkRepository,
RepositoryTwo: &ilk.DripFileIlkRepository{},
}
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
})
}) })

5
pkg/transformers/drip_file/repo/repository.go
View File

@ -16,7 +16,6 @@ package repo
import ( import (
"fmt" "fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -64,10 +63,6 @@ func (repository DripFileRepoRepository) MarkHeaderChecked(headerID int64) error
return shared.MarkHeaderChecked(headerID, repository.db, constants.DripFileRepoChecked) return shared.MarkHeaderChecked(headerID, repository.db, constants.DripFileRepoChecked)
} }
func (repository DripFileRepoRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DripFileRepoChecked)
}
func (repository *DripFileRepoRepository) SetDB(db *postgres.DB) { func (repository *DripFileRepoRepository) SetDB(db *postgres.DB) {
repository.db = db repository.db = db
} }

9
pkg/transformers/drip_file/repo/repository_test.go
View File

@ -82,13 +82,4 @@ var _ = Describe("Drip file repo repository", func() {
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
}) })
Describe("MissingHeaders", func() {
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
Repository: &dripFileRepoRepository,
RepositoryTwo: &repo.DripFileRepoRepository{},
}
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
})
}) })

5
pkg/transformers/drip_file/vow/repository.go
View File

@ -16,7 +16,6 @@ package vow
import ( import (
"fmt" "fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -63,10 +62,6 @@ func (repository DripFileVowRepository) MarkHeaderChecked(headerID int64) error
return shared.MarkHeaderChecked(headerID, repository.db, constants.DripFileVowChecked) return shared.MarkHeaderChecked(headerID, repository.db, constants.DripFileVowChecked)
} }
func (repository DripFileVowRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DripFileVowChecked)
}
func (repository *DripFileVowRepository) SetDB(db *postgres.DB) { func (repository *DripFileVowRepository) SetDB(db *postgres.DB) {
repository.db = db repository.db = db
} }

9
pkg/transformers/drip_file/vow/repository_test.go
View File

@ -74,15 +74,6 @@ var _ = Describe("Drip file vow repository", func() {
}) })
Describe("MarkHeaderChecked", func() { Describe("MarkHeaderChecked", func() {
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
Repository: &dripFileVowRepository,
RepositoryTwo: &vow.DripFileVowRepository{},
}
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
})
Describe("MissingHeaders", func() {
inputs := shared_behaviors.MarkedHeaderCheckedBehaviorInputs{ inputs := shared_behaviors.MarkedHeaderCheckedBehaviorInputs{
CheckedHeaderColumnName: constants.DripFileVowChecked, CheckedHeaderColumnName: constants.DripFileVowChecked,
Repository: &dripFileVowRepository, Repository: &dripFileVowRepository,

71
pkg/transformers/factories/log_note_transformer.go
View File

@ -15,10 +15,9 @@
package factories package factories
import ( import (
"github.com/ethereum/go-ethereum/core/types"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/ethereum/go-ethereum/common"
"github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
@ -28,58 +27,44 @@ type LogNoteTransformer struct {
Config shared.TransformerConfig Config shared.TransformerConfig
Converter LogNoteConverter Converter LogNoteConverter
Repository Repository Repository Repository
Fetcher shared.SettableLogFetcher
} }
func (transformer LogNoteTransformer) NewLogNoteTransformer(db *postgres.DB, bc core.BlockChain) shared.Transformer { func (transformer LogNoteTransformer) NewLogNoteTransformer(db *postgres.DB) shared.Transformer {
transformer.Repository.SetDB(db) transformer.Repository.SetDB(db)
transformer.Fetcher.SetBC(bc)
return transformer return transformer
} }
func (transformer LogNoteTransformer) Execute() error { func (transformer LogNoteTransformer) Execute(logs []types.Log, header core.Header) error {
transformerName := transformer.Config.TransformerName transformerName := transformer.Config.TransformerName
missingHeaders, err := transformer.Repository.MissingHeaders(transformer.Config.StartingBlockNumber, transformer.Config.EndingBlockNumber)
// No matching logs, mark the header as checked for this type of logs
if len(logs) < 1 {
err := transformer.Repository.MarkHeaderChecked(header.Id)
if err != nil {
log.Printf("Error marking header as checked in %v: %v", transformerName, err)
return err
}
return nil
}
models, err := transformer.Converter.ToModels(logs)
if err != nil { if err != nil {
log.Printf("Error fetching mising headers in %v transformer: %v", transformerName, err) log.Printf("Error converting logs in %v: %v", transformerName, err)
return err return err
} }
// Grab event signature from transformer config err = transformer.Repository.Create(header.Id, models)
// (Double-array structure required for go-ethereum FilterQuery) if err != nil {
var topic = [][]common.Hash{{common.HexToHash(transformer.Config.Topic)}} log.Printf("Error persisting %v record: %v", transformerName, err)
return err
log.Printf("Fetching %v event logs for %d headers", transformerName, len(missingHeaders))
for _, header := range missingHeaders {
// Fetch the missing logs for a given header
matchingLogs, err := transformer.Fetcher.FetchLogs(transformer.Config.ContractAddresses, topic, header)
if err != nil {
log.Printf("Error fetching matching logs in %v transformer: %v", transformerName, err)
return err
}
// No matching logs, mark the header as checked for this type of logs
if len(matchingLogs) < 1 {
err := transformer.Repository.MarkHeaderChecked(header.Id)
if err != nil {
log.Printf("Error marking header as checked in %v: %v", transformerName, err)
return err
}
// Continue with the next header; nothing to persist
continue
}
models, err := transformer.Converter.ToModels(matchingLogs)
if err != nil {
log.Printf("Error converting logs in %v: %v", transformerName, err)
return err
}
err = transformer.Repository.Create(header.Id, models)
if err != nil {
log.Printf("Error persisting %v record: %v", transformerName, err)
return err
}
} }
return nil return nil
} }
func (transformer LogNoteTransformer) GetName() string {
return transformer.Config.TransformerName
}
func (transformer LogNoteTransformer) GetConfig() shared.TransformerConfig {
return transformer.Config
}

86
pkg/transformers/factories/log_note_transformer_test.go
View File

@ -15,7 +15,7 @@
package factories_test package factories_test
import ( import (
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types"
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/core"
@ -31,9 +31,7 @@ var _ = Describe("LogNoteTransformer", func() {
var ( var (
repository mocks.MockRepository repository mocks.MockRepository
converter mocks.MockLogNoteConverter converter mocks.MockLogNoteConverter
fetcher mocks.MockLogFetcher
headerOne core.Header headerOne core.Header
headerTwo core.Header
transformer shared.Transformer transformer shared.Transformer
model test_data.GenericModel model test_data.GenericModel
config = test_data.GenericTestConfig config = test_data.GenericTestConfig
@ -43,75 +41,28 @@ var _ = Describe("LogNoteTransformer", func() {
BeforeEach(func() { BeforeEach(func() {
repository = mocks.MockRepository{} repository = mocks.MockRepository{}
converter = mocks.MockLogNoteConverter{} converter = mocks.MockLogNoteConverter{}
fetcher = mocks.MockLogFetcher{}
transformer = factories.LogNoteTransformer{ transformer = factories.LogNoteTransformer{
Config: config, Config: config,
Converter: &converter, Converter: &converter,
Repository: &repository, Repository: &repository,
Fetcher: &fetcher, }.NewLogNoteTransformer(nil)
}.NewLogNoteTransformer(nil, nil)
headerOne = core.Header{Id: rand.Int63(), BlockNumber: rand.Int63()} headerOne = core.Header{Id: rand.Int63(), BlockNumber: rand.Int63()}
headerTwo = core.Header{Id: rand.Int63(), BlockNumber: rand.Int63()}
}) })
It("sets the blockchain and database", func() { It("sets the database", func() {
Expect(fetcher.SetBcCalled).To(BeTrue())
Expect(repository.SetDbCalled).To(BeTrue()) Expect(repository.SetDbCalled).To(BeTrue())
}) })
It("gets missing headers for block numbers specified in config", func() { It("marks header checked if no logs are provided", func() {
err := transformer.Execute() err := transformer.Execute([]types.Log{}, headerOne)
Expect(err).NotTo(HaveOccurred())
Expect(repository.PassedStartingBlockNumber).To(Equal(config.StartingBlockNumber))
Expect(repository.PassedEndingBlockNumber).To(Equal(config.EndingBlockNumber))
})
It("returns error if repository returns error for missing headers", func() {
repository.SetMissingHeadersError(fakes.FakeError)
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("fetches logs for missing headers", func() {
repository.SetMissingHeaders([]core.Header{headerOne, headerTwo})
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(fetcher.FetchedBlocks).To(Equal([]int64{headerOne.BlockNumber, headerTwo.BlockNumber}))
Expect(fetcher.FetchedContractAddresses).To(Equal([][]string{config.ContractAddresses, config.ContractAddresses}))
expectedTopic := common.HexToHash(config.Topic)
Expect(fetcher.FetchedTopics).To(Equal([][]common.Hash{{expectedTopic}}))
})
It("returns error if fetcher returns error", func() {
repository.SetMissingHeaders([]core.Header{headerOne})
fetcher.SetFetcherError(fakes.FakeError)
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("marks header checked if no logs returned", func() {
repository.SetMissingHeaders([]core.Header{headerOne})
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
repository.AssertMarkHeaderCheckedCalledWith(headerOne.Id) repository.AssertMarkHeaderCheckedCalledWith(headerOne.Id)
}) })
It("doesn't attempt to convert or persist an empty collection when there are no logs", func() { It("doesn't attempt to convert or persist an empty collection when there are no logs", func() {
repository.SetMissingHeaders([]core.Header{headerOne, headerTwo}) err := transformer.Execute([]types.Log{}, headerOne)
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(converter.ToModelsCalledCounter).To(Equal(0)) Expect(converter.ToModelsCalledCounter).To(Equal(0))
@ -119,30 +70,23 @@ var _ = Describe("LogNoteTransformer", func() {
}) })
It("does not call repository.MarkCheckedHeader when there are logs", func() { It("does not call repository.MarkCheckedHeader when there are logs", func() {
repository.SetMissingHeaders([]core.Header{headerOne}) err := transformer.Execute(logs, headerOne)
fetcher.SetFetchedLogs(logs)
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
repository.AssertMarkHeaderCheckedNotCalled() repository.AssertMarkHeaderCheckedNotCalled()
}) })
It("returns error if marking header checked returns err", func() { It("returns error if marking header checked returns err", func() {
repository.SetMissingHeaders([]core.Header{headerOne})
repository.SetMarkHeaderCheckedError(fakes.FakeError) repository.SetMarkHeaderCheckedError(fakes.FakeError)
err := transformer.Execute() err := transformer.Execute([]types.Log{}, headerOne)
Expect(err).To(HaveOccurred()) Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError)) Expect(err).To(MatchError(fakes.FakeError))
}) })
It("converts matching logs to models", func() { It("converts matching logs to models", func() {
fetcher.SetFetchedLogs(logs) err := transformer.Execute(logs, headerOne)
repository.SetMissingHeaders([]core.Header{headerOne})
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(converter.PassedLogs).To(Equal(logs)) Expect(converter.PassedLogs).To(Equal(logs))
@ -150,20 +94,16 @@ var _ = Describe("LogNoteTransformer", func() {
It("returns error if converter returns error", func() { It("returns error if converter returns error", func() {
converter.SetConverterError(fakes.FakeError) converter.SetConverterError(fakes.FakeError)
fetcher.SetFetchedLogs(logs)
repository.SetMissingHeaders([]core.Header{headerOne})
err := transformer.Execute() err := transformer.Execute(logs, headerOne)
Expect(err).To(HaveOccurred()) Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError)) Expect(err).To(MatchError(fakes.FakeError))
}) })
It("persists the model", func() { It("persists the model", func() {
fetcher.SetFetchedLogs(logs)
repository.SetMissingHeaders([]core.Header{headerOne})
converter.SetReturnModels([]interface{}{model}) converter.SetReturnModels([]interface{}{model})
err := transformer.Execute() err := transformer.Execute(logs, headerOne)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(repository.PassedHeaderID).To(Equal(headerOne.Id)) Expect(repository.PassedHeaderID).To(Equal(headerOne.Id))
@ -171,11 +111,9 @@ var _ = Describe("LogNoteTransformer", func() {
}) })
It("returns error if repository returns error for create", func() { It("returns error if repository returns error for create", func() {
fetcher.SetFetchedLogs(logs)
repository.SetMissingHeaders([]core.Header{headerOne})
repository.SetCreateError(fakes.FakeError) repository.SetCreateError(fakes.FakeError)
err := transformer.Execute() err := transformer.Execute(logs, headerOne)
Expect(err).To(HaveOccurred()) Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError)) Expect(err).To(MatchError(fakes.FakeError))

2
pkg/transformers/factories/repository.go
View File

@ -15,13 +15,11 @@
package factories package factories
import ( import (
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
) )
type Repository interface { type Repository interface {
Create(headerID int64, models []interface{}) error Create(headerID int64, models []interface{}) error
MarkHeaderChecked(headerID int64) error MarkHeaderChecked(headerID int64) error
MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
SetDB(db *postgres.DB) SetDB(db *postgres.DB)
} }

75
pkg/transformers/factories/transformer.go
View File

@ -15,10 +15,9 @@
package factories package factories
import ( import (
"github.com/ethereum/go-ethereum/core/types"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/ethereum/go-ethereum/common"
"github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
@ -28,61 +27,51 @@ type Transformer struct {
Config shared.TransformerConfig Config shared.TransformerConfig
Converter Converter Converter Converter
Repository Repository Repository Repository
Fetcher shared.SettableLogFetcher
} }
func (transformer Transformer) NewTransformer(db *postgres.DB, bc core.BlockChain) shared.Transformer { func (transformer Transformer) NewTransformer(db *postgres.DB) shared.Transformer {
transformer.Repository.SetDB(db) transformer.Repository.SetDB(db)
transformer.Fetcher.SetBC(bc)
return transformer return transformer
} }
func (transformer Transformer) Execute() error { func (transformer Transformer) Execute(logs []types.Log, header core.Header) error {
transformerName := transformer.Config.TransformerName transformerName := transformer.Config.TransformerName
config := transformer.Config config := transformer.Config
topics := [][]common.Hash{{common.HexToHash(config.Topic)}}
missingHeaders, err := transformer.Repository.MissingHeaders(config.StartingBlockNumber, config.EndingBlockNumber) if len(logs) < 1 {
err := transformer.Repository.MarkHeaderChecked(header.Id)
if err != nil {
log.Printf("Error marking header as checked in %v: %v", transformerName, err)
return err
}
return nil
}
entities, err := transformer.Converter.ToEntities(config.ContractAbi, logs)
if err != nil { if err != nil {
log.Printf("Error fetching missing headers in %v transformer: %v \n", transformerName, err) log.Printf("Error converting logs to entities in %v: %v", transformerName, err)
return err return err
} }
log.Printf("Fetching %v event logs for %d headers \n", transformerName, len(missingHeaders))
for _, header := range missingHeaders {
logs, err := transformer.Fetcher.FetchLogs(config.ContractAddresses, topics, header)
if err != nil {
log.Printf("Error fetching matching logs in %v transformer: %v", transformerName, err)
return err
}
if len(logs) < 1 { models, err := transformer.Converter.ToModels(entities)
err = transformer.Repository.MarkHeaderChecked(header.Id) if err != nil {
if err != nil { log.Printf("Error converting entities to models in %v: %v", transformerName, err)
log.Printf("Error marking header as checked in %v: %v", transformerName, err) return err
return err }
}
continue
}
entities, err := transformer.Converter.ToEntities(config.ContractAbi, logs)
if err != nil {
log.Printf("Error converting logs to entities in %v: %v", transformerName, err)
return err
}
models, err := transformer.Converter.ToModels(entities)
if err != nil {
log.Printf("Error converting entities to models in %v: %v", transformerName, err)
return err
}
err = transformer.Repository.Create(header.Id, models)
if err != nil {
log.Printf("Error persisting %v record: %v", transformerName, err)
return err
}
err = transformer.Repository.Create(header.Id, models)
if err != nil {
log.Printf("Error persisting %v record: %v", transformerName, err)
return err
} }
return nil return nil
} }
func (transformer Transformer) GetName() string {
return transformer.Config.TransformerName
}
func (transformer Transformer) GetConfig() shared.TransformerConfig {
return transformer.Config
}

91
pkg/transformers/factories/transformer_test.go
View File

@ -15,7 +15,7 @@
package factories_test package factories_test
import ( import (
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types"
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/core"
@ -30,86 +30,39 @@ import (
var _ = Describe("Transformer", func() { var _ = Describe("Transformer", func() {
var ( var (
repository mocks.MockRepository repository mocks.MockRepository
fetcher mocks.MockLogFetcher
converter mocks.MockConverter converter mocks.MockConverter
transformer shared.Transformer transformer shared.Transformer
headerOne core.Header headerOne core.Header
headerTwo core.Header
config = test_data.GenericTestConfig config = test_data.GenericTestConfig
logs = test_data.GenericTestLogs logs = test_data.GenericTestLogs
) )
BeforeEach(func() { BeforeEach(func() {
repository = mocks.MockRepository{} repository = mocks.MockRepository{}
fetcher = mocks.MockLogFetcher{}
converter = mocks.MockConverter{} converter = mocks.MockConverter{}
transformer = factories.Transformer{ transformer = factories.Transformer{
Repository: &repository, Repository: &repository,
Fetcher: &fetcher,
Converter: &converter, Converter: &converter,
Config: config, Config: config,
}.NewTransformer(nil, nil) }.NewTransformer(nil)
headerOne = core.Header{Id: rand.Int63(), BlockNumber: rand.Int63()} headerOne = core.Header{Id: rand.Int63(), BlockNumber: rand.Int63()}
headerTwo = core.Header{Id: rand.Int63(), BlockNumber: rand.Int63()}
}) })
It("sets the blockchain and db", func() { It("sets the db", func() {
Expect(fetcher.SetBcCalled).To(BeTrue())
Expect(repository.SetDbCalled).To(BeTrue()) Expect(repository.SetDbCalled).To(BeTrue())
}) })
It("gets missing headers for blocks in the configured range", func() {
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(repository.PassedStartingBlockNumber).To(Equal(config.StartingBlockNumber))
Expect(repository.PassedEndingBlockNumber).To(Equal(config.EndingBlockNumber))
})
It("returns an error if it fails to get missing headers", func() {
repository.SetMissingHeadersError(fakes.FakeError)
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("fetches eth logs for each missing header", func() {
repository.SetMissingHeaders([]core.Header{headerOne, headerTwo})
expectedTopics := [][]common.Hash{{common.HexToHash(config.Topic)}}
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(fetcher.FetchedBlocks).To(Equal([]int64{headerOne.BlockNumber, headerTwo.BlockNumber}))
Expect(fetcher.FetchedTopics).To(Equal(expectedTopics))
Expect(fetcher.FetchedContractAddresses).To(Equal([][]string{config.ContractAddresses, config.ContractAddresses}))
})
It("returns an error if fetching logs fails", func() {
repository.SetMissingHeaders([]core.Header{headerOne})
fetcher.SetFetcherError(fakes.FakeError)
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("marks header checked if no logs returned", func() { It("marks header checked if no logs returned", func() {
headerID := int64(123) err := transformer.Execute([]types.Log{}, headerOne)
repository.SetMissingHeaders([]core.Header{{Id: headerID}})
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
repository.AssertMarkHeaderCheckedCalledWith(headerID) repository.AssertMarkHeaderCheckedCalledWith(headerOne.Id)
}) })
It("doesn't attempt to convert or persist an empty collection when there are no logs", func() { It("doesn't attempt to convert or persist an empty collection when there are no logs", func() {
repository.SetMissingHeaders([]core.Header{headerOne, headerTwo}) err := transformer.Execute([]types.Log{}, headerOne)
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(converter.ToEntitiesCalledCounter).To(Equal(0)) Expect(converter.ToEntitiesCalledCounter).To(Equal(0))
@ -118,29 +71,23 @@ var _ = Describe("Transformer", func() {
}) })
It("does not call repository.MarkCheckedHeader when there are logs", func() { It("does not call repository.MarkCheckedHeader when there are logs", func() {
repository.SetMissingHeaders([]core.Header{headerOne}) err := transformer.Execute(logs, headerOne)
fetcher.SetFetchedLogs(logs)
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
repository.AssertMarkHeaderCheckedNotCalled() repository.AssertMarkHeaderCheckedNotCalled()
}) })
It("returns error if marking header checked returns err", func() { It("returns error if marking header checked returns err", func() {
repository.SetMissingHeaders([]core.Header{headerOne})
repository.SetMarkHeaderCheckedError(fakes.FakeError) repository.SetMarkHeaderCheckedError(fakes.FakeError)
err := transformer.Execute() err := transformer.Execute([]types.Log{}, headerOne)
Expect(err).To(HaveOccurred()) Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError)) Expect(err).To(MatchError(fakes.FakeError))
}) })
It("converts an eth log to an entity", func() { It("converts an eth log to an entity", func() {
repository.SetMissingHeaders([]core.Header{headerOne}) err := transformer.Execute(logs, headerOne)
fetcher.SetFetchedLogs(logs)
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(converter.ContractAbi).To(Equal(config.ContractAbi)) Expect(converter.ContractAbi).To(Equal(config.ContractAbi))
@ -148,45 +95,37 @@ var _ = Describe("Transformer", func() {
}) })
It("returns an error if converter fails", func() { It("returns an error if converter fails", func() {
repository.SetMissingHeaders([]core.Header{headerOne})
fetcher.SetFetchedLogs(logs)
converter.ToEntitiesError = fakes.FakeError converter.ToEntitiesError = fakes.FakeError
err := transformer.Execute() err := transformer.Execute(logs, headerOne)
Expect(err).To(HaveOccurred()) Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError)) Expect(err).To(MatchError(fakes.FakeError))
}) })
It("converts an entity to a model", func() { It("converts an entity to a model", func() {
repository.SetMissingHeaders([]core.Header{headerOne})
fetcher.SetFetchedLogs(logs)
converter.EntitiesToReturn = []interface{}{test_data.GenericEntity{}} converter.EntitiesToReturn = []interface{}{test_data.GenericEntity{}}
err := transformer.Execute() err := transformer.Execute(logs, headerOne)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(converter.EntitiesToConvert[0]).To(Equal(test_data.GenericEntity{})) Expect(converter.EntitiesToConvert[0]).To(Equal(test_data.GenericEntity{}))
}) })
It("returns an error if converting to models fails", func() { It("returns an error if converting to models fails", func() {
repository.SetMissingHeaders([]core.Header{headerOne})
fetcher.SetFetchedLogs(logs)
converter.EntitiesToReturn = []interface{}{test_data.GenericEntity{}} converter.EntitiesToReturn = []interface{}{test_data.GenericEntity{}}
converter.ToModelsError = fakes.FakeError converter.ToModelsError = fakes.FakeError
err := transformer.Execute() err := transformer.Execute(logs, headerOne)
Expect(err).To(HaveOccurred()) Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError)) Expect(err).To(MatchError(fakes.FakeError))
}) })
It("persists the record", func() { It("persists the record", func() {
repository.SetMissingHeaders([]core.Header{headerOne})
fetcher.SetFetchedLogs(logs)
converter.ModelsToReturn = []interface{}{test_data.GenericModel{}} converter.ModelsToReturn = []interface{}{test_data.GenericModel{}}
err := transformer.Execute() err := transformer.Execute(logs, headerOne)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(repository.PassedHeaderID).To(Equal(headerOne.Id)) Expect(repository.PassedHeaderID).To(Equal(headerOne.Id))
@ -194,10 +133,8 @@ var _ = Describe("Transformer", func() {
}) })
It("returns error if persisting the record fails", func() { It("returns error if persisting the record fails", func() {
repository.SetMissingHeaders([]core.Header{headerOne})
fetcher.SetFetchedLogs(logs)
repository.SetCreateError(fakes.FakeError) repository.SetCreateError(fakes.FakeError)
err := transformer.Execute() err := transformer.Execute(logs, headerOne)
Expect(err).To(HaveOccurred()) Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError)) Expect(err).To(MatchError(fakes.FakeError))

5
pkg/transformers/flap_kick/repository.go
View File

@ -16,7 +16,6 @@ package flap_kick
import ( import (
"fmt" "fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -60,10 +59,6 @@ func (repository *FlapKickRepository) MarkHeaderChecked(headerID int64) error {
return shared.MarkHeaderChecked(headerID, repository.db, constants.FlapKickChecked) return shared.MarkHeaderChecked(headerID, repository.db, constants.FlapKickChecked)
} }
func (repository FlapKickRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.FlapKickChecked)
}
func (repository *FlapKickRepository) SetDB(db *postgres.DB) { func (repository *FlapKickRepository) SetDB(db *postgres.DB) {
repository.db = db repository.db = db
} }

9
pkg/transformers/flap_kick/repository_test.go
View File

@ -87,13 +87,4 @@ var _ = Describe("Flap Kick Repository", func() {
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
}) })
Describe("MissingHeaders", func() {
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
Repository: &flapKickRepository,
RepositoryTwo: &flap_kick.FlapKickRepository{},
}
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
})
}) })

5
pkg/transformers/flip_kick/repository.go
View File

@ -17,7 +17,6 @@ package flip_kick
import ( import (
"fmt" "fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -60,10 +59,6 @@ func (repository FlipKickRepository) MarkHeaderChecked(headerId int64) error {
return shared.MarkHeaderChecked(headerId, repository.db, constants.FlipKickChecked) return shared.MarkHeaderChecked(headerId, repository.db, constants.FlipKickChecked)
} }
func (repository FlipKickRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.FlipKickChecked)
}
func (repository *FlipKickRepository) SetDB(db *postgres.DB) { func (repository *FlipKickRepository) SetDB(db *postgres.DB) {
repository.db = db repository.db = db
} }

9
pkg/transformers/flip_kick/repository_test.go
View File

@ -87,15 +87,6 @@ var _ = Describe("FlipKick Repository", func() {
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
}) })
Describe("missing headers", func() {
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
Repository: &flipKickRepository,
RepositoryTwo: &flip_kick.FlipKickRepository{},
}
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
})
}) })
func assertDBRecordCount(db *postgres.DB, dbTable string, expectedCount int) { func assertDBRecordCount(db *postgres.DB, dbTable string, expectedCount int) {

5
pkg/transformers/flop_kick/repository.go
View File

@ -16,7 +16,6 @@ package flop_kick
import ( import (
"fmt" "fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -62,10 +61,6 @@ func (repository FlopKickRepository) MarkHeaderChecked(headerId int64) error {
return shared.MarkHeaderChecked(headerId, repository.db, constants.FlopKickChecked) return shared.MarkHeaderChecked(headerId, repository.db, constants.FlopKickChecked)
} }
func (repository FlopKickRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.FlopKickChecked)
}
func (repository *FlopKickRepository) SetDB(db *postgres.DB) { func (repository *FlopKickRepository) SetDB(db *postgres.DB) {
repository.db = db repository.db = db
} }

9
pkg/transformers/flop_kick/repository_test.go
View File

@ -85,13 +85,4 @@ var _ = Describe("FlopRepository", func() {
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
}) })
Describe("MissingHeaders", func() {
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
Repository: &repository,
RepositoryTwo: &flop_kick.FlopKickRepository{},
}
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
})
}) })

5
pkg/transformers/frob/repository.go
View File

@ -17,7 +17,6 @@ package frob
import ( import (
"fmt" "fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -59,10 +58,6 @@ func (repository FrobRepository) MarkHeaderChecked(headerID int64) error {
return shared.MarkHeaderChecked(headerID, repository.db, constants.FrobChecked) return shared.MarkHeaderChecked(headerID, repository.db, constants.FrobChecked)
} }
func (repository FrobRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.FrobChecked)
}
func (repository *FrobRepository) SetDB(db *postgres.DB) { func (repository *FrobRepository) SetDB(db *postgres.DB) {
repository.db = db repository.db = db
} }

9
pkg/transformers/frob/repository_test.go
View File

@ -86,13 +86,4 @@ var _ = Describe("Frob repository", func() {
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
}) })
Describe("MissingHeaders", func() {
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
Repository: &frobRepository,
RepositoryTwo: &frob.FrobRepository{},
}
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
})
}) })

15
pkg/transformers/integration_tests/bite.go
View File

@ -45,17 +45,24 @@ var _ = Describe("Bite Transformer", func() {
db := test_config.NewTestDB(blockChain.Node()) db := test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db) test_config.CleanTestDB(db)
err = persistHeader(db, blockNumber, blockChain) header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
initializer := factories.Transformer{ initializer := factories.Transformer{
Config: config, Config: config,
Converter: &bite.BiteConverter{}, Converter: &bite.BiteConverter{},
Repository: &bite.BiteRepository{}, Repository: &bite.BiteRepository{},
Fetcher: &shared.Fetcher{},
} }
transformer := initializer.NewTransformer(db, blockChain) transformer := initializer.NewTransformer(db)
err = transformer.Execute()
fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(
[]common.Address{common.HexToAddress(config.ContractAddresses[0])},
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []bite.BiteModel var dbResult []bite.BiteModel

47
pkg/transformers/integration_tests/cat_file.go
View File

@ -15,6 +15,7 @@
package integration_tests package integration_tests
import ( import (
"github.com/ethereum/go-ethereum/common"
"github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"sort" "sort"
@ -39,6 +40,7 @@ var _ = Describe("Cat File transformer", func() {
rpcClient client.RpcClient rpcClient client.RpcClient
err error err error
ethClient *ethclient.Client ethClient *ethclient.Client
fetcher *shared.Fetcher
) )
BeforeEach(func() { BeforeEach(func() {
@ -48,13 +50,15 @@ var _ = Describe("Cat File transformer", func() {
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
db = test_config.NewTestDB(blockChain.Node()) db = test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db) test_config.CleanTestDB(db)
fetcher = shared.NewFetcher(blockChain)
}) })
// Cat contract Kovan address: 0x2f34f22a00ee4b7a8f8bbc4eaee1658774c624e0 // Cat contract Kovan address: 0x2f34f22a00ee4b7a8f8bbc4eaee1658774c624e0
It("persists a chop lump event", func() { It("persists a chop lump event", func() {
// transaction: 0x98574bfba4d05c3875be10d2376e678d005dbebe9a4520363407508fd21f4014 // transaction: 0x98574bfba4d05c3875be10d2376e678d005dbebe9a4520363407508fd21f4014
chopLumpBlockNumber := int64(8762253) chopLumpBlockNumber := int64(8762253)
err = persistHeader(db, chopLumpBlockNumber, blockChain) header, err := persistHeader(db, chopLumpBlockNumber, blockChain)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
config := chop_lump.CatFileChopLumpConfig config := chop_lump.CatFileChopLumpConfig
@ -65,10 +69,16 @@ var _ = Describe("Cat File transformer", func() {
Config: config, Config: config,
Converter: &chop_lump.CatFileChopLumpConverter{}, Converter: &chop_lump.CatFileChopLumpConverter{},
Repository: &chop_lump.CatFileChopLumpRepository{}, Repository: &chop_lump.CatFileChopLumpRepository{},
Fetcher: &shared.Fetcher{},
} }
transformer := initializer.NewLogNoteTransformer(db, blockChain) transformer := initializer.NewLogNoteTransformer(db)
err := transformer.Execute()
logs, err := fetcher.FetchLogs(
[]common.Address{common.HexToAddress(config.ContractAddresses[0])},
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []chop_lump.CatFileChopLumpModel var dbResult []chop_lump.CatFileChopLumpModel
@ -92,7 +102,7 @@ var _ = Describe("Cat File transformer", func() {
It("persists a flip event", func() { It("persists a flip event", func() {
// transaction: 0x44bc18fdb1a5a263db114e7879653304db3e19ceb4e4496f21bc0a76c5faccbe // transaction: 0x44bc18fdb1a5a263db114e7879653304db3e19ceb4e4496f21bc0a76c5faccbe
flipBlockNumber := int64(8751794) flipBlockNumber := int64(8751794)
err = persistHeader(db, flipBlockNumber, blockChain) header, err := persistHeader(db, flipBlockNumber, blockChain)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
config := flip.CatFileFlipConfig config := flip.CatFileFlipConfig
@ -103,10 +113,17 @@ var _ = Describe("Cat File transformer", func() {
Config: config, Config: config,
Converter: &flip.CatFileFlipConverter{}, Converter: &flip.CatFileFlipConverter{},
Repository: &flip.CatFileFlipRepository{}, Repository: &flip.CatFileFlipRepository{},
Fetcher: &shared.Fetcher{},
} }
transformer := initializer.NewLogNoteTransformer(db, blockChain)
err := transformer.Execute() transformer := initializer.NewLogNoteTransformer(db)
logs, err := fetcher.FetchLogs(
[]common.Address{common.HexToAddress(config.ContractAddresses[0])},
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []flip.CatFileFlipModel var dbResult []flip.CatFileFlipModel
@ -122,7 +139,7 @@ var _ = Describe("Cat File transformer", func() {
It("persists a pit vow event", func() { It("persists a pit vow event", func() {
// transaction: 0x44bc18fdb1a5a263db114e7879653304db3e19ceb4e4496f21bc0a76c5faccbe // transaction: 0x44bc18fdb1a5a263db114e7879653304db3e19ceb4e4496f21bc0a76c5faccbe
pitVowBlockNumber := int64(8751794) pitVowBlockNumber := int64(8751794)
err = persistHeader(db, pitVowBlockNumber, blockChain) header, err := persistHeader(db, pitVowBlockNumber, blockChain)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
config := pit_vow.CatFilePitVowConfig config := pit_vow.CatFilePitVowConfig
@ -133,10 +150,16 @@ var _ = Describe("Cat File transformer", func() {
Config: config, Config: config,
Converter: &pit_vow.CatFilePitVowConverter{}, Converter: &pit_vow.CatFilePitVowConverter{},
Repository: &pit_vow.CatFilePitVowRepository{}, Repository: &pit_vow.CatFilePitVowRepository{},
Fetcher: &shared.Fetcher{},
} }
transformer := initializer.NewLogNoteTransformer(db, blockChain) transformer := initializer.NewLogNoteTransformer(db)
err := transformer.Execute()
logs, err := fetcher.FetchLogs(
[]common.Address{common.HexToAddress(config.ContractAddresses[0])},
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []pit_vow.CatFilePitVowModel var dbResult []pit_vow.CatFilePitVowModel

63
pkg/transformers/integration_tests/deal.go
View File

@ -15,6 +15,7 @@
package integration_tests package integration_tests
import ( import (
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
@ -29,8 +30,13 @@ import (
var _ = Describe("Deal transformer", func() { var _ = Describe("Deal transformer", func() {
var ( var (
db *postgres.DB db *postgres.DB
blockChain core.BlockChain blockChain core.BlockChain
config shared.TransformerConfig
initializer factories.LogNoteTransformer
fetcher *shared.Fetcher
addresses []common.Address
topics []common.Hash
) )
BeforeEach(func() { BeforeEach(func() {
@ -40,26 +46,35 @@ var _ = Describe("Deal transformer", func() {
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
db = test_config.NewTestDB(blockChain.Node()) db = test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db) test_config.CleanTestDB(db)
config = deal.DealConfig
initializer = factories.LogNoteTransformer{
Config: config,
Converter: &deal.DealConverter{},
Repository: &deal.DealRepository{},
}
fetcher = shared.NewFetcher(blockChain)
addresses = shared.HexStringsToAddresses(config.ContractAddresses)
topics = []common.Hash{common.HexToHash(config.Topic)}
}) })
It("persists a flip deal log event", func() { It("persists a flip deal log event", func() {
// transaction: 0x05b5eabac2ace136f0f7e0efc61d7d42abe8e8938cc0f04fbf1a6ba545d59e58 // transaction: 0x05b5eabac2ace136f0f7e0efc61d7d42abe8e8938cc0f04fbf1a6ba545d59e58
flipBlockNumber := int64(8958007) flipBlockNumber := int64(8958007)
err := persistHeader(db, flipBlockNumber, blockChain) header, err := persistHeader(db, flipBlockNumber, blockChain)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
config := deal.DealConfig initializer.Config.StartingBlockNumber = flipBlockNumber
config.StartingBlockNumber = flipBlockNumber initializer.Config.EndingBlockNumber = flipBlockNumber
config.EndingBlockNumber = flipBlockNumber
initializer := factories.LogNoteTransformer{ logs, err := fetcher.FetchLogs(addresses, topics, header)
Config: config, Expect(err).NotTo(HaveOccurred())
Converter: &deal.DealConverter{},
Repository: &deal.DealRepository{}, transformer := initializer.NewLogNoteTransformer(db)
Fetcher: &shared.Fetcher{}, err = transformer.Execute(logs, header)
}
transformer := initializer.NewLogNoteTransformer(db, blockChain)
err = transformer.Execute()
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []deal.DealModel var dbResult []deal.DealModel
@ -77,21 +92,17 @@ var _ = Describe("Deal transformer", func() {
It("persists a flap deal log event", func() { It("persists a flap deal log event", func() {
flapBlockNumber := int64(9004628) flapBlockNumber := int64(9004628)
err := persistHeader(db, flapBlockNumber, blockChain) header, err := persistHeader(db, flapBlockNumber, blockChain)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
config := deal.DealConfig initializer.Config.StartingBlockNumber = flapBlockNumber
config.StartingBlockNumber = flapBlockNumber initializer.Config.EndingBlockNumber = flapBlockNumber
config.EndingBlockNumber = flapBlockNumber
initializer := factories.LogNoteTransformer{ logs, err := fetcher.FetchLogs(addresses, topics, header)
Config: config, Expect(err).NotTo(HaveOccurred())
Converter: &deal.DealConverter{},
Repository: &deal.DealRepository{}, transformer := initializer.NewLogNoteTransformer(db)
Fetcher: &shared.Fetcher{}, err = transformer.Execute(logs, header)
}
transformer := initializer.NewLogNoteTransformer(db, blockChain)
err = transformer.Execute()
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []deal.DealModel var dbResult []deal.DealModel

42
pkg/transformers/integration_tests/dent.go
View File

@ -1,6 +1,7 @@
package integration_tests package integration_tests
import ( import (
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -15,8 +16,14 @@ import (
var _ = Describe("Dent transformer", func() { var _ = Describe("Dent transformer", func() {
var ( var (
db *postgres.DB db *postgres.DB
blockChain core.BlockChain blockChain core.BlockChain
fetcher *shared.Fetcher
transformer shared.Transformer
config shared.TransformerConfig
addresses []common.Address
topics []common.Hash
initializer factories.LogNoteTransformer
) )
BeforeEach(func() { BeforeEach(func() {
@ -26,25 +33,32 @@ var _ = Describe("Dent transformer", func() {
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
db = test_config.NewTestDB(blockChain.Node()) db = test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db) test_config.CleanTestDB(db)
config = dent.DentConfig
addresses = shared.HexStringsToAddresses(config.ContractAddresses)
topics = []common.Hash{common.HexToHash(config.Topic)}
fetcher = shared.NewFetcher(blockChain)
initializer = factories.LogNoteTransformer{
Config: config,
Converter: &dent.DentConverter{},
Repository: &dent.DentRepository{},
}
}) })
It("persists a flop dent log event", func() { It("persists a flop dent log event", func() {
blockNumber := int64(8955613) blockNumber := int64(8955613)
err := persistHeader(db, blockNumber, blockChain) header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
config := dent.DentConfig initializer.Config.StartingBlockNumber = blockNumber
config.StartingBlockNumber = blockNumber initializer.Config.EndingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber
initializer := factories.LogNoteTransformer{ logs, err := fetcher.FetchLogs(addresses, topics, header)
Config: config, Expect(err).NotTo(HaveOccurred())
Converter: &dent.DentConverter{},
Repository: &dent.DentRepository{}, transformer = initializer.NewLogNoteTransformer(db)
Fetcher: &shared.Fetcher{}, err = transformer.Execute(logs, header)
}
transformer := initializer.NewLogNoteTransformer(db, blockChain)
err = transformer.Execute()
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []dent.DentModel var dbResult []dent.DentModel

38
pkg/transformers/integration_tests/drip_drip.go
View File

@ -15,8 +15,11 @@
package integration_tests package integration_tests
import ( import (
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
@ -25,32 +28,43 @@ import (
) )
var _ = Describe("DripDrip Transformer", func() { var _ = Describe("DripDrip Transformer", func() {
var (
db *postgres.DB
blockChain core.BlockChain
)
BeforeEach(func() {
rpcClient, ethClient, err := getClients(ipc)
Expect(err).NotTo(HaveOccurred())
blockChain, err = getBlockChain(rpcClient, ethClient)
Expect(err).NotTo(HaveOccurred())
db = test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
})
It("transforms DripDrip log events", func() { It("transforms DripDrip log events", func() {
blockNumber := int64(8934775) blockNumber := int64(8934775)
config := drip_drip.DripDripConfig config := drip_drip.DripDripConfig
config.StartingBlockNumber = blockNumber config.StartingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber config.EndingBlockNumber = blockNumber
rpcClient, ethClient, err := getClients(ipc) header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
blockChain, err := getBlockChain(rpcClient, ethClient)
Expect(err).NotTo(HaveOccurred())
db := test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
err = persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
initializer := factories.LogNoteTransformer{ initializer := factories.LogNoteTransformer{
Config: config, Config: config,
Converter: &drip_drip.DripDripConverter{}, Converter: &drip_drip.DripDripConverter{},
Repository: &drip_drip.DripDripRepository{}, Repository: &drip_drip.DripDripRepository{},
Fetcher: &shared.Fetcher{},
} }
transformer := initializer.NewLogNoteTransformer(db, blockChain) transformer := initializer.NewLogNoteTransformer(db)
err = transformer.Execute() fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResults []drip_drip.DripDripModel var dbResults []drip_drip.DripDripModel

40
pkg/transformers/integration_tests/drip_file_vow.go
View File

@ -15,8 +15,11 @@
package integration_tests package integration_tests
import ( import (
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
@ -25,31 +28,44 @@ import (
) )
var _ = Describe("Drip File Vow LogNoteTransformer", func() { var _ = Describe("Drip File Vow LogNoteTransformer", func() {
var (
db *postgres.DB
blockChain core.BlockChain
)
BeforeEach(func() {
rpcClient, ethClient, err := getClients(ipc)
Expect(err).NotTo(HaveOccurred())
blockChain, err = getBlockChain(rpcClient, ethClient)
Expect(err).NotTo(HaveOccurred())
db = test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
})
It("transforms DripFileVow log events", func() { It("transforms DripFileVow log events", func() {
blockNumber := int64(8762197) blockNumber := int64(8762197)
config := vow.DripFileVowConfig config := vow.DripFileVowConfig
config.StartingBlockNumber = blockNumber config.StartingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber config.EndingBlockNumber = blockNumber
rpcClient, ethClient, err := getClients(ipc) header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
blockChain, err := getBlockChain(rpcClient, ethClient)
Expect(err).NotTo(HaveOccurred())
db := test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
err = persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
initializer := factories.LogNoteTransformer{ initializer := factories.LogNoteTransformer{
Config: config, Config: config,
Fetcher: &shared.Fetcher{},
Converter: &vow.DripFileVowConverter{}, Converter: &vow.DripFileVowConverter{},
Repository: &vow.DripFileVowRepository{}, Repository: &vow.DripFileVowRepository{},
} }
transformer := initializer.NewLogNoteTransformer(db, blockChain) transformer := initializer.NewLogNoteTransformer(db)
err = transformer.Execute()
fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []vow.DripFileVowModel var dbResult []vow.DripFileVowModel

45
pkg/transformers/integration_tests/flap_kick.go
View File

@ -15,6 +15,10 @@
package integration_tests package integration_tests
import ( import (
"github.com/ethereum/go-ethereum/common"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/test_config"
"time" "time"
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
@ -23,35 +27,46 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/flap_kick" "github.com/vulcanize/vulcanizedb/pkg/transformers/flap_kick"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/test_config"
) )
var _ = Describe("FlapKick Transformer", func() { var _ = Describe("FlapKick Transformer", func() {
var (
db *postgres.DB
blockChain core.BlockChain
)
BeforeEach(func() {
rpcClient, ethClient, err := getClients(ipc)
Expect(err).NotTo(HaveOccurred())
blockChain, err = getBlockChain(rpcClient, ethClient)
Expect(err).NotTo(HaveOccurred())
db = test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
})
It("fetches and transforms a FlapKick event from Kovan chain", func() { It("fetches and transforms a FlapKick event from Kovan chain", func() {
blockNumber := int64(9002933) blockNumber := int64(9002933)
config := flap_kick.FlapKickConfig config := flap_kick.FlapKickConfig
config.StartingBlockNumber = blockNumber config.StartingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber config.EndingBlockNumber = blockNumber
rpcClient, ethClient, err := getClients(ipc) header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
blockChain, err := getBlockChain(rpcClient, ethClient)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
db := test_config.NewTestDB(blockChain.Node()) transformer := factories.Transformer{
test_config.CleanTestDB(db)
err = persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
initializer := factories.Transformer{
Config: config, Config: config,
Converter: &flap_kick.FlapKickConverter{}, Converter: &flap_kick.FlapKickConverter{},
Repository: &flap_kick.FlapKickRepository{}, Repository: &flap_kick.FlapKickRepository{},
Fetcher: &shared.Fetcher{}, }.NewTransformer(db)
}
transformer := initializer.NewTransformer(db, blockChain) fetcher := shared.NewFetcher(blockChain)
err = transformer.Execute() logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []flap_kick.FlapKickModel var dbResult []flap_kick.FlapKickModel

19
pkg/transformers/integration_tests/flip_kick.go
View File

@ -65,21 +65,26 @@ var _ = Describe("FlipKick Transformer", func() {
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
blockChain, err := getBlockChain(rpcClient, ethClient) blockChain, err := getBlockChain(rpcClient, ethClient)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
db := test_config.NewTestDB(blockChain.Node()) db := test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db) test_config.CleanTestDB(db)
err = persistHeader(db, blockNumber, blockChain) header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
initializer := factories.Transformer{ transformer := factories.Transformer{
Config: config, Config: config,
Converter: &flip_kick.FlipKickConverter{}, Converter: &flip_kick.FlipKickConverter{},
Repository: &flip_kick.FlipKickRepository{}, Repository: &flip_kick.FlipKickRepository{},
Fetcher: &shared.Fetcher{}, }.NewTransformer(db)
}
transformer := initializer.NewTransformer(db, blockChain) fetcher := shared.NewFetcher(blockChain)
err = transformer.Execute() logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []flip_kick.FlipKickModel var dbResult []flip_kick.FlipKickModel

61
pkg/transformers/integration_tests/flop_kick.go
View File

@ -35,8 +35,13 @@ import (
var _ = Describe("FlopKick Transformer", func() { var _ = Describe("FlopKick Transformer", func() {
var ( var (
db *postgres.DB db *postgres.DB
blockChain core.BlockChain blockChain core.BlockChain
config shared.TransformerConfig
initializer factories.Transformer
fetcher shared.LogFetcher
addresses []common.Address
topics []common.Hash
) )
BeforeEach(func() { BeforeEach(func() {
@ -46,25 +51,33 @@ var _ = Describe("FlopKick Transformer", func() {
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
db = test_config.NewTestDB(blockChain.Node()) db = test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db) test_config.CleanTestDB(db)
config = flop_kick.Config
initializer = factories.Transformer{
Config: config,
Converter: &flop_kick.FlopKickConverter{},
Repository: &flop_kick.FlopKickRepository{},
}
fetcher = shared.NewFetcher(blockChain)
addresses = shared.HexStringsToAddresses(config.ContractAddresses)
topics = []common.Hash{common.HexToHash(config.Topic)}
}) })
It("fetches and transforms a FlopKick event from Kovan chain", func() { It("fetches and transforms a FlopKick event from Kovan chain", func() {
blockNumber := int64(8672119) blockNumber := int64(8672119)
config := flop_kick.Config initializer.Config.StartingBlockNumber = blockNumber
config.StartingBlockNumber = blockNumber initializer.Config.EndingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber
err := persistHeader(db, blockNumber, blockChain) header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
initializer := factories.Transformer{ logs, err := fetcher.FetchLogs(addresses, topics, header)
Config: config, Expect(err).NotTo(HaveOccurred())
Converter: &flop_kick.FlopKickConverter{},
Repository: &flop_kick.FlopKickRepository{}, transformer := initializer.NewTransformer(db)
Fetcher: &shared.Fetcher{}, err = transformer.Execute(logs, header)
}
transformer := initializer.NewTransformer(db, blockChain)
err = transformer.Execute()
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []flop_kick.Model var dbResult []flop_kick.Model
@ -82,21 +95,17 @@ var _ = Describe("FlopKick Transformer", func() {
It("fetches and transforms another FlopKick event from Kovan chain", func() { It("fetches and transforms another FlopKick event from Kovan chain", func() {
blockNumber := int64(8955611) blockNumber := int64(8955611)
config := flop_kick.Config initializer.Config.StartingBlockNumber = blockNumber
config.StartingBlockNumber = blockNumber initializer.Config.EndingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber
err := persistHeader(db, blockNumber, blockChain) header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
initializer := factories.Transformer{ logs, err := fetcher.FetchLogs(addresses, topics, header)
Config: config, Expect(err).NotTo(HaveOccurred())
Converter: &flop_kick.FlopKickConverter{},
Repository: &flop_kick.FlopKickRepository{}, transformer := initializer.NewTransformer(db)
Fetcher: &shared.Fetcher{}, err = transformer.Execute(logs, header)
}
transformer := initializer.NewTransformer(db, blockChain)
err = transformer.Execute()
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []flop_kick.Model var dbResult []flop_kick.Model

38
pkg/transformers/integration_tests/frob.go
View File

@ -33,8 +33,11 @@ import (
var _ = Describe("Frob Transformer", func() { var _ = Describe("Frob Transformer", func() {
var ( var (
db *postgres.DB db *postgres.DB
blockChain core.BlockChain blockChain core.BlockChain
fetcher *shared.Fetcher
config shared.TransformerConfig
initializer factories.Transformer
) )
BeforeEach(func() { BeforeEach(func() {
@ -44,25 +47,32 @@ var _ = Describe("Frob Transformer", func() {
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
db = test_config.NewTestDB(blockChain.Node()) db = test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db) test_config.CleanTestDB(db)
fetcher = shared.NewFetcher(blockChain)
config = frob.FrobConfig
initializer = factories.Transformer{
Config: config,
Converter: &frob.FrobConverter{},
Repository: &frob.FrobRepository{},
}
}) })
It("fetches and transforms a Frob event from Kovan chain", func() { It("fetches and transforms a Frob event from Kovan chain", func() {
blockNumber := int64(8935258) blockNumber := int64(8935258)
config := frob.FrobConfig initializer.Config.StartingBlockNumber = blockNumber
config.StartingBlockNumber = blockNumber initializer.Config.EndingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber
err := persistHeader(db, blockNumber, blockChain) header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
initializer := factories.Transformer{ logs, err := fetcher.FetchLogs(
Config: config, shared.HexStringsToAddresses(config.ContractAddresses),
Converter: &frob.FrobConverter{}, []common.Hash{common.HexToHash(config.Topic)},
Repository: &frob.FrobRepository{}, header)
Fetcher: &shared.Fetcher{}, Expect(err).NotTo(HaveOccurred())
}
transformer := initializer.NewTransformer(db, blockChain) transformer := initializer.NewTransformer(db)
err = transformer.Execute() err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []frob.FrobModel var dbResult []frob.FrobModel

10
pkg/transformers/integration_tests/helpers.go
View File

@ -42,12 +42,14 @@ func getBlockChain(rpcClient client.RpcClient, ethClient *ethclient.Client) (cor
return blockChain, nil return blockChain, nil
} }
func persistHeader(db *postgres.DB, blockNumber int64, blockChain core.BlockChain) error { // Persist the header for a given block to postgres. Returns the header if successful.
func persistHeader(db *postgres.DB, blockNumber int64, blockChain core.BlockChain) (core.Header, error) {
header, err := blockChain.GetHeaderByNumber(blockNumber) header, err := blockChain.GetHeaderByNumber(blockNumber)
if err != nil { if err != nil {
return err return core.Header{}, err
} }
headerRepository := repositories.NewHeaderRepository(db) headerRepository := repositories.NewHeaderRepository(db)
_, err = headerRepository.CreateOrUpdateHeader(header) id, err := headerRepository.CreateOrUpdateHeader(header)
return err header.Id = id
return header, err
} }

36
pkg/transformers/integration_tests/pit_file_debt_ceiling.go
View File

@ -15,8 +15,11 @@
package integration_tests package integration_tests
import ( import (
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
@ -25,31 +28,44 @@ import (
) )
var _ = Describe("PitFileDebtCeiling LogNoteTransformer", func() { var _ = Describe("PitFileDebtCeiling LogNoteTransformer", func() {
var (
db *postgres.DB
blockChain core.BlockChain
)
BeforeEach(func() {
rpcClient, ethClient, err := getClients(ipc)
Expect(err).NotTo(HaveOccurred())
blockChain, err = getBlockChain(rpcClient, ethClient)
Expect(err).NotTo(HaveOccurred())
db = test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
})
It("fetches and transforms a PitFileDebtCeiling event from Kovan chain", func() { It("fetches and transforms a PitFileDebtCeiling event from Kovan chain", func() {
blockNumber := int64(8535578) blockNumber := int64(8535578)
config := debt_ceiling.DebtCeilingFileConfig config := debt_ceiling.DebtCeilingFileConfig
config.StartingBlockNumber = blockNumber config.StartingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber config.EndingBlockNumber = blockNumber
rpcClient, ethClient, err := getClients(ipc) header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
blockChain, err := getBlockChain(rpcClient, ethClient)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
db := test_config.NewTestDB(blockChain.Node()) fetcher := shared.NewFetcher(blockChain)
test_config.CleanTestDB(db) logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
err = persistHeader(db, blockNumber, blockChain) []common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
initializer := factories.LogNoteTransformer{ initializer := factories.LogNoteTransformer{
Config: config, Config: config,
Fetcher: &shared.Fetcher{},
Converter: &debt_ceiling.PitFileDebtCeilingConverter{}, Converter: &debt_ceiling.PitFileDebtCeilingConverter{},
Repository: &debt_ceiling.PitFileDebtCeilingRepository{}, Repository: &debt_ceiling.PitFileDebtCeilingRepository{},
} }
transformer := initializer.NewLogNoteTransformer(db, blockChain) transformer := initializer.NewLogNoteTransformer(db)
err = transformer.Execute()
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []debt_ceiling.PitFileDebtCeilingModel var dbResult []debt_ceiling.PitFileDebtCeilingModel

72
pkg/transformers/integration_tests/pit_file_ilk.go
View File

@ -15,8 +15,11 @@
package integration_tests package integration_tests
import ( import (
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/ilk" "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/ilk"
@ -25,31 +28,47 @@ import (
) )
var _ = Describe("PitFileIlk LogNoteTransformer", func() { var _ = Describe("PitFileIlk LogNoteTransformer", func() {
It("fetches and transforms a Pit.file ilk 'spot' event from Kovan", func() { var (
blockNumber := int64(9103223) db *postgres.DB
config := ilk.IlkFileConfig blockChain core.BlockChain
config.StartingBlockNumber = blockNumber initializer factories.LogNoteTransformer
config.EndingBlockNumber = blockNumber addresses []common.Address
topics []common.Hash
)
BeforeEach(func() {
rpcClient, ethClient, err := getClients(ipc) rpcClient, ethClient, err := getClients(ipc)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
blockChain, err := getBlockChain(rpcClient, ethClient) blockChain, err = getBlockChain(rpcClient, ethClient)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
db = test_config.NewTestDB(blockChain.Node())
db := test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db) test_config.CleanTestDB(db)
config := ilk.IlkFileConfig
err = persistHeader(db, blockNumber, blockChain) addresses = shared.HexStringsToAddresses(config.ContractAddresses)
Expect(err).NotTo(HaveOccurred()) topics = []common.Hash{common.HexToHash(config.Topic)}
initializer := factories.LogNoteTransformer{ initializer = factories.LogNoteTransformer{
Config: config, Config: config,
Fetcher: &shared.Fetcher{},
Converter: &ilk.PitFileIlkConverter{}, Converter: &ilk.PitFileIlkConverter{},
Repository: &ilk.PitFileIlkRepository{}, Repository: &ilk.PitFileIlkRepository{},
} }
transformer := initializer.NewLogNoteTransformer(db, blockChain) })
err = transformer.Execute()
It("fetches and transforms a Pit.file ilk 'spot' event from Kovan", func() {
blockNumber := int64(9103223)
initializer.Config.StartingBlockNumber = blockNumber
initializer.Config.EndingBlockNumber = blockNumber
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(addresses, topics, header)
Expect(err).NotTo(HaveOccurred())
transformer := initializer.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []ilk.PitFileIlkModel var dbResult []ilk.PitFileIlkModel
@ -64,29 +83,18 @@ var _ = Describe("PitFileIlk LogNoteTransformer", func() {
It("fetches and transforms a Pit.file ilk 'line' event from Kovan", func() { It("fetches and transforms a Pit.file ilk 'line' event from Kovan", func() {
blockNumber := int64(8762253) blockNumber := int64(8762253)
config := ilk.IlkFileConfig initializer.Config.StartingBlockNumber = blockNumber
config.StartingBlockNumber = blockNumber initializer.Config.EndingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber
rpcClient, ethClient, err := getClients(ipc) header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
blockChain, err := getBlockChain(rpcClient, ethClient)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
db := test_config.NewTestDB(blockChain.Node()) fetcher := shared.NewFetcher(blockChain)
test_config.CleanTestDB(db) logs, err := fetcher.FetchLogs(addresses, topics, header)
err = persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
initializer := factories.LogNoteTransformer{ transformer := initializer.NewLogNoteTransformer(db)
Config: config, err = transformer.Execute(logs, header)
Fetcher: &shared.Fetcher{},
Converter: &ilk.PitFileIlkConverter{},
Repository: &ilk.PitFileIlkRepository{},
}
transformer := initializer.NewLogNoteTransformer(db, blockChain)
err = transformer.Execute()
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []ilk.PitFileIlkModel var dbResult []ilk.PitFileIlkModel

107
pkg/transformers/integration_tests/price_feeds.go
View File

@ -15,6 +15,7 @@
package integration_tests package integration_tests
import ( import (
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
@ -29,8 +30,12 @@ import (
var _ = Describe("Price feeds transformer", func() { var _ = Describe("Price feeds transformer", func() {
var ( var (
db *postgres.DB db *postgres.DB
blockChain core.BlockChain blockChain core.BlockChain
config shared.TransformerConfig
fetcher *shared.Fetcher
initializer factories.LogNoteTransformer
topics []common.Hash
) )
BeforeEach(func() { BeforeEach(func() {
@ -40,86 +45,94 @@ var _ = Describe("Price feeds transformer", func() {
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
db = test_config.NewTestDB(blockChain.Node()) db = test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db) test_config.CleanTestDB(db)
config = price_feeds.PriceFeedConfig
topics = []common.Hash{common.HexToHash(config.Topic)}
fetcher = shared.NewFetcher(blockChain)
initializer = factories.LogNoteTransformer{
Config: config,
Converter: &price_feeds.PriceFeedConverter{},
Repository: &price_feeds.PriceFeedRepository{},
}
}) })
It("persists a ETH/USD price feed event", func() { It("persists a ETH/USD price feed event", func() {
blockNumber := int64(8763054) blockNumber := int64(8763054)
err := persistHeader(db, blockNumber, blockChain) header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
config := price_feeds.PriceFeedConfig addresses := []string{constants.PipContractAddress}
config.ContractAddresses = []string{constants.PipContractAddress} initializer.Config.ContractAddresses = addresses
config.StartingBlockNumber = blockNumber initializer.Config.StartingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber initializer.Config.EndingBlockNumber = blockNumber
transformerInitializer := factories.LogNoteTransformer{ logs, err := fetcher.FetchLogs(
Config: config, shared.HexStringsToAddresses(addresses),
Converter: &price_feeds.PriceFeedConverter{}, topics,
Repository: &price_feeds.PriceFeedRepository{}, header)
Fetcher: &shared.Fetcher{}, Expect(err).NotTo(HaveOccurred())
}
transformer := transformerInitializer.NewLogNoteTransformer(db, blockChain)
err = transformer.Execute() transformer := initializer.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var model price_feeds.PriceFeedModel var model price_feeds.PriceFeedModel
err = db.Get(&model, `SELECT block_number, medianizer_address, usd_value, tx_idx, raw_log FROM maker.price_feeds WHERE block_number = $1`, config.StartingBlockNumber) err = db.Get(&model, `SELECT block_number, medianizer_address, usd_value, tx_idx, raw_log FROM maker.price_feeds WHERE block_number = $1`, initializer.Config.StartingBlockNumber)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(model.UsdValue).To(Equal("207.314891143000011198")) Expect(model.UsdValue).To(Equal("207.314891143000011198"))
Expect(model.MedianizerAddress).To(Equal(config.ContractAddresses[0])) Expect(model.MedianizerAddress).To(Equal(addresses[0]))
}) })
It("persists a MKR/USD price feed event", func() { It("persists a MKR/USD price feed event", func() {
blockNumber := int64(8763059) blockNumber := int64(8763059)
err := persistHeader(db, blockNumber, blockChain) header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
config := price_feeds.PriceFeedConfig addresses := []string{constants.PepContractAddress}
config.ContractAddresses = []string{constants.PepContractAddress} initializer.Config.ContractAddresses = addresses
config.StartingBlockNumber = blockNumber initializer.Config.StartingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber initializer.Config.EndingBlockNumber = blockNumber
transformerInitializer := factories.LogNoteTransformer{ logs, err := fetcher.FetchLogs(
Config: config, shared.HexStringsToAddresses(addresses),
Converter: &price_feeds.PriceFeedConverter{}, topics,
Repository: &price_feeds.PriceFeedRepository{}, header)
Fetcher: &shared.Fetcher{}, Expect(err).NotTo(HaveOccurred())
}
transformer := transformerInitializer.NewLogNoteTransformer(db, blockChain)
err = transformer.Execute() transformer := initializer.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var model price_feeds.PriceFeedModel var model price_feeds.PriceFeedModel
err = db.Get(&model, `SELECT block_number, medianizer_address, usd_value, tx_idx, raw_log FROM maker.price_feeds WHERE block_number = $1`, config.StartingBlockNumber) err = db.Get(&model, `SELECT block_number, medianizer_address, usd_value, tx_idx, raw_log FROM maker.price_feeds WHERE block_number = $1`, initializer.Config.StartingBlockNumber)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(model.UsdValue).To(Equal("391.803979212000001553")) Expect(model.UsdValue).To(Equal("391.803979212000001553"))
Expect(model.MedianizerAddress).To(Equal(config.ContractAddresses[0])) Expect(model.MedianizerAddress).To(Equal(addresses[0]))
}) })
It("persists a REP/USD price feed event", func() { It("persists a REP/USD price feed event", func() {
blockNumber := int64(8763062) blockNumber := int64(8763062)
err := persistHeader(db, blockNumber, blockChain) header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
config := price_feeds.PriceFeedConfig addresses := []string{constants.RepContractAddress}
config.ContractAddresses = []string{constants.RepContractAddress} initializer.Config.ContractAddresses = addresses
config.StartingBlockNumber = blockNumber initializer.Config.StartingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber initializer.Config.EndingBlockNumber = blockNumber
transformerInitializer := factories.LogNoteTransformer{ logs, err := fetcher.FetchLogs(
Config: config, shared.HexStringsToAddresses(addresses),
Converter: &price_feeds.PriceFeedConverter{}, topics,
Repository: &price_feeds.PriceFeedRepository{}, header)
Fetcher: &shared.Fetcher{}, Expect(err).NotTo(HaveOccurred())
}
transformer := transformerInitializer.NewLogNoteTransformer(db, blockChain)
err = transformer.Execute() transformer := initializer.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var model price_feeds.PriceFeedModel var model price_feeds.PriceFeedModel
err = db.Get(&model, `SELECT block_number, medianizer_address, usd_value, tx_idx, raw_log FROM maker.price_feeds WHERE block_number = $1`, config.StartingBlockNumber) err = db.Get(&model, `SELECT block_number, medianizer_address, usd_value, tx_idx, raw_log FROM maker.price_feeds WHERE block_number = $1`, initializer.Config.StartingBlockNumber)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(model.UsdValue).To(Equal("12.816928482699999847")) Expect(model.UsdValue).To(Equal("12.816928482699999847"))
Expect(model.MedianizerAddress).To(Equal(config.ContractAddresses[0])) Expect(model.MedianizerAddress).To(Equal(addresses[0]))
}) })
}) })

82
pkg/transformers/integration_tests/tend.go
View File

@ -15,6 +15,7 @@
package integration_tests package integration_tests
import ( import (
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/core"
@ -28,8 +29,13 @@ import (
var _ = Describe("Tend LogNoteTransformer", func() { var _ = Describe("Tend LogNoteTransformer", func() {
var ( var (
db *postgres.DB db *postgres.DB
blockChain core.BlockChain blockChain core.BlockChain
config shared.TransformerConfig
fetcher *shared.Fetcher
initializer factories.LogNoteTransformer
addresses []common.Address
topics []common.Hash
) )
BeforeEach(func() { BeforeEach(func() {
@ -39,25 +45,33 @@ var _ = Describe("Tend LogNoteTransformer", func() {
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
db = test_config.NewTestDB(blockChain.Node()) db = test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db) test_config.CleanTestDB(db)
config = tend.TendConfig
fetcher = shared.NewFetcher(blockChain)
addresses = shared.HexStringsToAddresses(config.ContractAddresses)
topics = []common.Hash{common.HexToHash(config.Topic)}
initializer = factories.LogNoteTransformer{
Config: config,
Converter: &tend.TendConverter{},
Repository: &tend.TendRepository{},
}
}) })
It("fetches and transforms a Flip Tend event from Kovan chain", func() { It("fetches and transforms a Flip Tend event from Kovan chain", func() {
blockNumber := int64(8935601) blockNumber := int64(8935601)
config := tend.TendConfig initializer.Config.StartingBlockNumber = blockNumber
config.StartingBlockNumber = blockNumber initializer.Config.EndingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber
err := persistHeader(db, blockNumber, blockChain) header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
initializer := factories.LogNoteTransformer{ logs, err := fetcher.FetchLogs(addresses, topics, header)
Config: config, Expect(err).NotTo(HaveOccurred())
Fetcher: &shared.Fetcher{},
Converter: &tend.TendConverter{}, transformer := initializer.NewLogNoteTransformer(db)
Repository: &tend.TendRepository{}, err = transformer.Execute(logs, header)
}
transformer := initializer.NewLogNoteTransformer(db, blockChain)
err = transformer.Execute()
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []tend.TendModel var dbResult []tend.TendModel
@ -80,21 +94,17 @@ var _ = Describe("Tend LogNoteTransformer", func() {
It("fetches and transforms a subsequent Flip Tend event from Kovan chain for the same auction", func() { It("fetches and transforms a subsequent Flip Tend event from Kovan chain for the same auction", func() {
blockNumber := int64(8935731) blockNumber := int64(8935731)
config := tend.TendConfig initializer.Config.StartingBlockNumber = blockNumber
config.StartingBlockNumber = blockNumber initializer.Config.EndingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber
err := persistHeader(db, blockNumber, blockChain) header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
initializer := factories.LogNoteTransformer{ logs, err := fetcher.FetchLogs(addresses, topics, header)
Config: config, Expect(err).NotTo(HaveOccurred())
Fetcher: &shared.Fetcher{},
Converter: &tend.TendConverter{}, transformer := initializer.NewLogNoteTransformer(db)
Repository: &tend.TendRepository{}, err = transformer.Execute(logs, header)
}
transformer := initializer.NewLogNoteTransformer(db, blockChain)
err = transformer.Execute()
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []tend.TendModel var dbResult []tend.TendModel
@ -117,21 +127,17 @@ var _ = Describe("Tend LogNoteTransformer", func() {
It("fetches and transforms a Flap Tend event from the Kovan chain", func() { It("fetches and transforms a Flap Tend event from the Kovan chain", func() {
blockNumber := int64(9003177) blockNumber := int64(9003177)
config := tend.TendConfig initializer.Config.StartingBlockNumber = blockNumber
config.StartingBlockNumber = blockNumber initializer.Config.EndingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber
err := persistHeader(db, blockNumber, blockChain) header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
initializer := factories.LogNoteTransformer{ logs, err := fetcher.FetchLogs(addresses, topics, header)
Config: config, Expect(err).NotTo(HaveOccurred())
Fetcher: &shared.Fetcher{},
Converter: &tend.TendConverter{}, transformer := initializer.NewLogNoteTransformer(db)
Repository: &tend.TendRepository{}, err = transformer.Execute(logs, header)
}
transformer := initializer.NewLogNoteTransformer(db, blockChain)
err = transformer.Execute()
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []tend.TendModel var dbResult []tend.TendModel

16
pkg/transformers/integration_tests/vat_flux.go
View File

@ -15,6 +15,7 @@
package integration_tests package integration_tests
import ( import (
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
@ -39,17 +40,24 @@ var _ = Describe("VatFlux LogNoteTransformer", func() {
db := test_config.NewTestDB(blockChain.Node()) db := test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db) test_config.CleanTestDB(db)
err = persistHeader(db, blockNumber, blockChain) header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
initializer := factories.LogNoteTransformer{ initializer := factories.LogNoteTransformer{
Config: config, Config: config,
Fetcher: &shared.Fetcher{},
Converter: &vat_flux.VatFluxConverter{}, Converter: &vat_flux.VatFluxConverter{},
Repository: &vat_flux.VatFluxRepository{}, Repository: &vat_flux.VatFluxRepository{},
} }
transformer := initializer.NewLogNoteTransformer(db, blockChain) transformer := initializer.NewLogNoteTransformer(db)
err = transformer.Execute()
err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []vat_flux.VatFluxModel var dbResult []vat_flux.VatFluxModel

40
pkg/transformers/integration_tests/vat_fold.go
View File

@ -18,39 +18,53 @@ import (
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/test_config"
"github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/vat_fold" "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_fold"
"github.com/vulcanize/vulcanizedb/test_config"
) )
var _ = Describe("VatFold Transformer", func() { var _ = Describe("VatFold Transformer", func() {
var (
db *postgres.DB
blockChain core.BlockChain
)
BeforeEach(func() {
rpcClient, ethClient, err := getClients(ipc)
Expect(err).NotTo(HaveOccurred())
blockChain, err = getBlockChain(rpcClient, ethClient)
Expect(err).NotTo(HaveOccurred())
db = test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
})
It("transforms VatFold log events", func() { It("transforms VatFold log events", func() {
blockNumber := int64(9367233) blockNumber := int64(9367233)
config := vat_fold.VatFoldConfig config := vat_fold.VatFoldConfig
config.StartingBlockNumber = blockNumber config.StartingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber config.EndingBlockNumber = blockNumber
rpcClient, ethClient, err := getClients(ipc) header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
blockChain, err := getBlockChain(rpcClient, ethClient)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
db := test_config.NewTestDB(blockChain.Node()) fetcher := shared.NewFetcher(blockChain)
test_config.CleanTestDB(db) logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
err = persistHeader(db, blockNumber, blockChain) []common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
initializer := factories.LogNoteTransformer{ transformer := factories.LogNoteTransformer{
Config: config, Config: config,
Fetcher: &shared.Fetcher{},
Converter: &vat_fold.VatFoldConverter{}, Converter: &vat_fold.VatFoldConverter{},
Repository: &vat_fold.VatFoldRepository{}, Repository: &vat_fold.VatFoldRepository{},
} }.NewLogNoteTransformer(db)
transformer := initializer.NewLogNoteTransformer(db, blockChain)
err = transformer.Execute() err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResults []vat_fold.VatFoldModel var dbResults []vat_fold.VatFoldModel

15
pkg/transformers/integration_tests/vat_grab.go
View File

@ -15,6 +15,7 @@
package integration_tests package integration_tests
import ( import (
"github.com/ethereum/go-ethereum/common"
"math/big" "math/big"
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
@ -41,17 +42,23 @@ var _ = Describe("Vat Grab Transformer", func() {
db := test_config.NewTestDB(blockChain.Node()) db := test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db) test_config.CleanTestDB(db)
err = persistHeader(db, blockNumber, blockChain) header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
transformer := factories.LogNoteTransformer{ transformer := factories.LogNoteTransformer{
Config: config, Config: config,
Converter: &vat_grab.VatGrabConverter{}, Converter: &vat_grab.VatGrabConverter{},
Repository: &vat_grab.VatGrabRepository{}, Repository: &vat_grab.VatGrabRepository{},
Fetcher: &shared.Fetcher{}, }.NewLogNoteTransformer(db)
}.NewLogNoteTransformer(db, blockChain)
err = transformer.Execute() err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []vat_grab.VatGrabModel var dbResult []vat_grab.VatGrabModel

14
pkg/transformers/integration_tests/vat_heal.go
View File

@ -40,17 +40,23 @@ var _ = Describe("VatHeal Transformer", func() {
db := test_config.NewTestDB(blockChain.Node()) db := test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db) test_config.CleanTestDB(db)
err = persistHeader(db, blockNumber, blockChain) header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
transformer := factories.LogNoteTransformer{ transformer := factories.LogNoteTransformer{
Config: config, Config: config,
Converter: &vat_heal.VatHealConverter{}, Converter: &vat_heal.VatHealConverter{},
Repository: &vat_heal.VatHealRepository{}, Repository: &vat_heal.VatHealRepository{},
Fetcher: &shared.Fetcher{}, }.NewLogNoteTransformer(db)
}.NewLogNoteTransformer(db, blockChain)
err = transformer.Execute() err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResults []vat_heal.VatHealModel var dbResults []vat_heal.VatHealModel

19
pkg/transformers/integration_tests/vat_init.go
View File

@ -15,6 +15,7 @@
package integration_tests package integration_tests
import ( import (
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
@ -39,17 +40,23 @@ var _ = Describe("VatInit LogNoteTransformer", func() {
db := test_config.NewTestDB(blockChain.Node()) db := test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db) test_config.CleanTestDB(db)
err = persistHeader(db, blockNumber, blockChain) header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
initializer := factories.LogNoteTransformer{ fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
transformer := factories.LogNoteTransformer{
Config: config, Config: config,
Fetcher: &shared.Fetcher{},
Converter: &vat_init.VatInitConverter{}, Converter: &vat_init.VatInitConverter{},
Repository: &vat_init.VatInitRepository{}, Repository: &vat_init.VatInitRepository{},
} }.NewLogNoteTransformer(db)
transformer := initializer.NewLogNoteTransformer(db, blockChain)
err = transformer.Execute() err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResults []vat_init.VatInitModel var dbResults []vat_init.VatInitModel

18
pkg/transformers/integration_tests/vat_move.go
View File

@ -40,17 +40,23 @@ var _ = Describe("VatMove LogNoteTransformer", func() {
db := test_config.NewTestDB(blockChain.Node()) db := test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db) test_config.CleanTestDB(db)
err = persistHeader(db, blockNumber, blockChain) header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
initializer := factories.LogNoteTransformer{ fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
transformer := factories.LogNoteTransformer{
Config: config, Config: config,
Fetcher: &shared.Fetcher{},
Converter: &vat_move.VatMoveConverter{}, Converter: &vat_move.VatMoveConverter{},
Repository: &vat_move.VatMoveRepository{}, Repository: &vat_move.VatMoveRepository{},
} }.NewLogNoteTransformer(db)
transformer := initializer.NewLogNoteTransformer(db, blockChain)
err = transformer.Execute() err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResults []vat_move.VatMoveModel var dbResults []vat_move.VatMoveModel

22
pkg/transformers/integration_tests/vat_slip.go
View File

@ -1,6 +1,7 @@
package integration_tests package integration_tests
import ( import (
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
@ -18,31 +19,38 @@ var _ = Describe("Vat slip transformer", func() {
blockChain core.BlockChain blockChain core.BlockChain
) )
It("persists vat slip event", func() { BeforeEach(func() {
rpcClient, ethClient, err := getClients(ipc) rpcClient, ethClient, err := getClients(ipc)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
blockChain, err = getBlockChain(rpcClient, ethClient) blockChain, err = getBlockChain(rpcClient, ethClient)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
db = test_config.NewTestDB(blockChain.Node()) db = test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db) test_config.CleanTestDB(db)
})
It("persists vat slip event", func() {
blockNumber := int64(8953655) blockNumber := int64(8953655)
config := vat_slip.VatSlipConfig config := vat_slip.VatSlipConfig
config.StartingBlockNumber = blockNumber config.StartingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber config.EndingBlockNumber = blockNumber
err = persistHeader(db, blockNumber, blockChain) header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
initializer := factories.LogNoteTransformer{ fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
transformer := factories.LogNoteTransformer{
Config: config, Config: config,
Fetcher: &shared.Fetcher{},
Converter: &vat_slip.VatSlipConverter{}, Converter: &vat_slip.VatSlipConverter{},
Repository: &vat_slip.VatSlipRepository{}, Repository: &vat_slip.VatSlipRepository{},
} }.NewLogNoteTransformer(db)
transformer := initializer.NewLogNoteTransformer(db, blockChain)
err = transformer.Execute() err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var headerID int64 var headerID int64

19
pkg/transformers/integration_tests/vat_tune.go
View File

@ -15,6 +15,7 @@
package integration_tests package integration_tests
import ( import (
"github.com/ethereum/go-ethereum/common"
"math/big" "math/big"
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
@ -41,17 +42,23 @@ var _ = Describe("VatTune LogNoteTransformer", func() {
db := test_config.NewTestDB(blockChain.Node()) db := test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db) test_config.CleanTestDB(db)
err = persistHeader(db, blockNumber, blockChain) header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
initializer := factories.LogNoteTransformer{ fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
transformer := factories.LogNoteTransformer{
Config: config, Config: config,
Fetcher: &shared.Fetcher{},
Converter: &vat_tune.VatTuneConverter{}, Converter: &vat_tune.VatTuneConverter{},
Repository: &vat_tune.VatTuneRepository{}, Repository: &vat_tune.VatTuneRepository{},
} }.NewLogNoteTransformer(db)
transformer := initializer.NewLogNoteTransformer(db, blockChain)
err = transformer.Execute() err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []vat_tune.VatTuneModel var dbResult []vat_tune.VatTuneModel

40
pkg/transformers/integration_tests/vow_flog.go
View File

@ -15,8 +15,11 @@
package integration_tests package integration_tests
import ( import (
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
@ -25,32 +28,43 @@ import (
) )
var _ = Describe("VowFlog LogNoteTransformer", func() { var _ = Describe("VowFlog LogNoteTransformer", func() {
var (
db *postgres.DB
blockChain core.BlockChain
)
BeforeEach(func() {
rpcClient, ethClient, err := getClients(ipc)
Expect(err).NotTo(HaveOccurred())
blockChain, err = getBlockChain(rpcClient, ethClient)
Expect(err).NotTo(HaveOccurred())
db = test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
})
It("transforms VowFlog log events", func() { It("transforms VowFlog log events", func() {
blockNumber := int64(8946819) blockNumber := int64(8946819)
config := vow_flog.VowFlogConfig config := vow_flog.VowFlogConfig
config.StartingBlockNumber = blockNumber config.StartingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber config.EndingBlockNumber = blockNumber
rpcClient, ethClient, err := getClients(ipc) header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
blockChain, err := getBlockChain(rpcClient, ethClient)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
db := test_config.NewTestDB(blockChain.Node()) fetcher := shared.NewFetcher(blockChain)
test_config.CleanTestDB(db) logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
err = persistHeader(db, blockNumber, blockChain) []common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(1).To(Equal(1))
initializer := factories.LogNoteTransformer{ transformer := factories.LogNoteTransformer{
Config: config, Config: config,
Fetcher: &shared.Fetcher{},
Converter: &vow_flog.VowFlogConverter{}, Converter: &vow_flog.VowFlogConverter{},
Repository: &vow_flog.VowFlogRepository{}, Repository: &vow_flog.VowFlogRepository{},
} }.NewLogNoteTransformer(db)
transformer := initializer.NewLogNoteTransformer(db, blockChain)
err = transformer.Execute() err = transformer.Execute(logs, header)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []vow_flog.VowFlogModel var dbResult []vow_flog.VowFlogModel

5
pkg/transformers/pit_file/debt_ceiling/repository.go
View File

@ -16,7 +16,6 @@ package debt_ceiling
import ( import (
"fmt" "fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -64,10 +63,6 @@ func (repository PitFileDebtCeilingRepository) MarkHeaderChecked(headerID int64)
return shared.MarkHeaderChecked(headerID, repository.db, constants.PitFileDebtCeilingChecked) return shared.MarkHeaderChecked(headerID, repository.db, constants.PitFileDebtCeilingChecked)
} }
func (repository PitFileDebtCeilingRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.PitFileDebtCeilingChecked)
}
func (repository *PitFileDebtCeilingRepository) SetDB(db *postgres.DB) { func (repository *PitFileDebtCeilingRepository) SetDB(db *postgres.DB) {
repository.db = db repository.db = db
} }

9
pkg/transformers/pit_file/debt_ceiling/repository_test.go
View File

@ -81,13 +81,4 @@ var _ = Describe("Pit file debt ceiling repository", func() {
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
}) })
Describe("MissingHeaders", func() {
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
Repository: &pitFileDebtCeilingRepository,
RepositoryTwo: &debt_ceiling.PitFileDebtCeilingRepository{},
}
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
})
}) })

5
pkg/transformers/pit_file/ilk/repository.go
View File

@ -16,7 +16,6 @@ package ilk
import ( import (
"fmt" "fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -62,10 +61,6 @@ func (repository PitFileIlkRepository) MarkHeaderChecked(headerID int64) error {
return shared.MarkHeaderChecked(headerID, repository.db, constants.PitFileIlkChecked) return shared.MarkHeaderChecked(headerID, repository.db, constants.PitFileIlkChecked)
} }
func (repository PitFileIlkRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.PitFileIlkChecked)
}
func (repository *PitFileIlkRepository) SetDB(db *postgres.DB) { func (repository *PitFileIlkRepository) SetDB(db *postgres.DB) {
repository.db = db repository.db = db
} }

9
pkg/transformers/pit_file/ilk/repository_test.go
View File

@ -82,13 +82,4 @@ var _ = Describe("Pit file ilk repository", func() {
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
}) })
Describe("MissingHeaders", func() {
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
Repository: &pitFileIlkRepository,
RepositoryTwo: &ilk.PitFileIlkRepository{},
}
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
})
}) })

5
pkg/transformers/price_feeds/repository.go
View File

@ -16,7 +16,6 @@ package price_feeds
import ( import (
"fmt" "fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -58,10 +57,6 @@ func (repository PriceFeedRepository) MarkHeaderChecked(headerID int64) error {
return shared.MarkHeaderChecked(headerID, repository.db, constants.PriceFeedsChecked) return shared.MarkHeaderChecked(headerID, repository.db, constants.PriceFeedsChecked)
} }
func (repository PriceFeedRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.PriceFeedsChecked)
}
func (repository *PriceFeedRepository) SetDB(db *postgres.DB) { func (repository *PriceFeedRepository) SetDB(db *postgres.DB) {
repository.db = db repository.db = db
} }

9
pkg/transformers/price_feeds/repository_test.go
View File

@ -82,13 +82,4 @@ var _ = Describe("Price feeds repository", func() {
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
}) })
Describe("MissingHeaders", func() {
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
Repository: &priceFeedRepository,
RepositoryTwo: &price_feeds.PriceFeedRepository{},
}
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
})
}) })

65
pkg/transformers/shared/log_chunker.go Normal file
View File

@ -0,0 +1,65 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package shared
import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
)
type Chunker interface {
AddConfigs(transformerConfigs []TransformerConfig)
ChunkLogs(logs []types.Log) map[string][]types.Log
}
type LogChunker struct {
AddressToNames map[string][]string
NameToTopic0 map[string]common.Hash
}
// Returns a new log chunker with initialised maps.
// Needs to have configs added with `AddConfigs` to consider logs for the respective transformer.
func NewLogChunker() *LogChunker {
return &LogChunker{
AddressToNames: map[string][]string{},
NameToTopic0: map[string]common.Hash{},
}
}
// Configures the chunker by adding more addreses and topics to consider.
func (chunker *LogChunker) AddConfigs(transformerConfigs []TransformerConfig) {
for _, config := range transformerConfigs {
for _, address := range config.ContractAddresses {
chunker.AddressToNames[address] = append(chunker.AddressToNames[address], config.TransformerName)
chunker.NameToTopic0[config.TransformerName] = common.HexToHash(config.Topic)
}
}
}
// Goes through an array of logs, associating relevant logs (matching addresses and topic) with transformers
func (chunker *LogChunker) ChunkLogs(logs []types.Log) map[string][]types.Log {
chunks := map[string][]types.Log{}
for _, log := range logs {
// Topic0 is not unique to each transformer, also need to consider the contract address
relevantTransformers := chunker.AddressToNames[log.Address.String()]
for _, transformer := range relevantTransformers {
if chunker.NameToTopic0[transformer] == log.Topics[0] {
chunks[transformer] = append(chunks[transformer], log)
}
}
}
return chunks
}

137
pkg/transformers/shared/log_chunker_test.go Normal file
View File

@ -0,0 +1,137 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package shared_test
import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
)
var _ = Describe("Log chunker", func() {
var (
configs []shared.TransformerConfig
chunker *shared.LogChunker
)
BeforeEach(func() {
configA := shared.TransformerConfig{
TransformerName: "TransformerA",
ContractAddresses: []string{"0x00000000000000000000000000000000000000A1", "0x00000000000000000000000000000000000000A2"},
Topic: "0xA",
}
configB := shared.TransformerConfig{
TransformerName: "TransformerB",
ContractAddresses: []string{"0x00000000000000000000000000000000000000B1"},
Topic: "0xB",
}
configC := shared.TransformerConfig{
TransformerName: "TransformerC",
ContractAddresses: []string{"0x00000000000000000000000000000000000000A2"},
Topic: "0xC",
}
configs = []shared.TransformerConfig{configA, configB, configC}
chunker = shared.NewLogChunker()
chunker.AddConfigs(configs)
})
Describe("initialisation", func() {
It("creates lookup maps correctly", func() {
Expect(chunker.AddressToNames).To(Equal(map[string][]string{
"0x00000000000000000000000000000000000000A1": []string{"TransformerA"},
"0x00000000000000000000000000000000000000A2": []string{"TransformerA", "TransformerC"},
"0x00000000000000000000000000000000000000B1": []string{"TransformerB"},
}))
Expect(chunker.NameToTopic0).To(Equal(map[string]common.Hash{
"TransformerA": common.HexToHash("0xA"),
"TransformerB": common.HexToHash("0xB"),
"TransformerC": common.HexToHash("0xC"),
}))
})
})
Describe("AddConfigs", func() {
It("can add more configs later", func() {
configD := shared.TransformerConfig{
TransformerName: "TransformerD",
ContractAddresses: []string{"0x000000000000000000000000000000000000000D"},
Topic: "0xD",
}
chunker.AddConfigs([]shared.TransformerConfig{configD})
Expect(chunker.AddressToNames).To(ContainElement([]string{"TransformerD"}))
Expect(chunker.NameToTopic0).To(ContainElement(common.HexToHash("0xD")))
})
})
Describe("ChunkLogs", func() {
It("only associates logs with relevant topic0 and address to transformers", func() {
logs := []types.Log{log1, log2, log3, log4, log5}
chunks := chunker.ChunkLogs(logs)
Expect(chunks["TransformerA"]).To(And(ContainElement(log1), ContainElement(log4)))
Expect(chunks["TransformerB"]).To(BeEmpty())
Expect(chunks["TransformerC"]).To(ContainElement(log5))
})
})
})
var (
// Match TransformerA
log1 = types.Log{
Address: common.HexToAddress("0xA1"),
Topics: []common.Hash{
common.HexToHash("0xA"),
common.HexToHash("0xLogTopic1"),
},
}
// Match TransformerA address, but not topic0
log2 = types.Log{
Address: common.HexToAddress("0xA1"),
Topics: []common.Hash{
common.HexToHash("0xB"),
common.HexToHash("0xLogTopic2"),
},
}
// Match TransformerA topic, but TransformerB address
log3 = types.Log{
Address: common.HexToAddress("0xB1"),
Topics: []common.Hash{
common.HexToHash("0xA"),
common.HexToHash("0xLogTopic3"),
},
}
// Match TransformerA, with the other address
log4 = types.Log{
Address: common.HexToAddress("0xA2"),
Topics: []common.Hash{
common.HexToHash("0xA"),
common.HexToHash("0xLogTopic4"),
},
}
// Match TransformerC, which shares address with TransformerA
log5 = types.Log{
Address: common.HexToAddress("0xA2"),
Topics: []common.Hash{
common.HexToHash("0xC"),
common.HexToHash("0xLogTopic5"),
},
}
)

39
pkg/transformers/shared/log_fetcher.go
View File

@ -23,45 +23,34 @@ import (
) )
type LogFetcher interface { type LogFetcher interface {
FetchLogs(contractAddresses []string, topics [][]common.Hash, header core.Header) ([]types.Log, error) FetchLogs(contractAddresses []common.Address, topics []common.Hash, missingHeader core.Header) ([]types.Log, error)
}
type SettableLogFetcher interface {
LogFetcher
SetBC(bc core.BlockChain)
} }
type Fetcher struct { type Fetcher struct {
blockChain core.BlockChain blockChain core.BlockChain
} }
func (fetcher *Fetcher) SetBC(bc core.BlockChain) { func NewFetcher(blockchain core.BlockChain) *Fetcher {
fetcher.blockChain = bc return &Fetcher{
}
func NewFetcher(blockchain core.BlockChain) Fetcher {
return Fetcher{
blockChain: blockchain, blockChain: blockchain,
} }
} }
func (fetcher Fetcher) FetchLogs(contractAddresses []string, topics [][]common.Hash, header core.Header) ([]types.Log, error) { // Checks all topic0s, on all addresses, fetching matching logs for the given header
addresses := hexStringsToAddresses(contractAddresses) func (fetcher Fetcher) FetchLogs(addresses []common.Address, topic0s []common.Hash, header core.Header) ([]types.Log, error) {
blockHash := common.HexToHash(header.Hash) blockHash := common.HexToHash(header.Hash)
query := ethereum.FilterQuery{ query := ethereum.FilterQuery{
BlockHash: &blockHash, BlockHash: &blockHash,
Addresses: addresses, Addresses: addresses,
Topics: topics, // Search for _any_ of the topics in topic0 position; see docs on `FilterQuery`
} Topics: [][]common.Hash{topic0s},
return fetcher.blockChain.GetEthLogsWithCustomQuery(query)
}
func hexStringsToAddresses(hexStrings []string) []common.Address {
var addresses []common.Address
for _, hexString := range hexStrings {
address := common.HexToAddress(hexString)
addresses = append(addresses, address)
} }
return addresses logs, err := fetcher.blockChain.GetEthLogsWithCustomQuery(query)
if err != nil {
// TODO review aggregate fetching error handling
return []types.Log{}, err
}
return logs, nil
} }

12
pkg/transformers/shared/log_fetcher_test.go
View File

@ -32,8 +32,12 @@ var _ = Describe("Fetcher", func() {
fetcher := shared.NewFetcher(blockChain) fetcher := shared.NewFetcher(blockChain)
header := fakes.FakeHeader header := fakes.FakeHeader
addresses := []string{"0xfakeAddress", "0xanotherFakeAddress"} addresses := []common.Address{
topicZeros := [][]common.Hash{{common.BytesToHash([]byte{1, 2, 3, 4, 5})}} common.HexToAddress("0xfakeAddress"),
common.HexToAddress("0xanotherFakeAddress"),
}
topicZeros := []common.Hash{common.BytesToHash([]byte{1, 2, 3, 4, 5})}
_, err := fetcher.FetchLogs(addresses, topicZeros, header) _, err := fetcher.FetchLogs(addresses, topicZeros, header)
@ -45,7 +49,7 @@ var _ = Describe("Fetcher", func() {
expectedQuery := ethereum.FilterQuery{ expectedQuery := ethereum.FilterQuery{
BlockHash: &blockHash, BlockHash: &blockHash,
Addresses: []common.Address{address1, address2}, Addresses: []common.Address{address1, address2},
Topics: topicZeros, Topics: [][]common.Hash{topicZeros},
} }
blockChain.AssertGetEthLogsWithCustomQueryCalledWith(expectedQuery) blockChain.AssertGetEthLogsWithCustomQueryCalledWith(expectedQuery)
}) })
@ -55,7 +59,7 @@ var _ = Describe("Fetcher", func() {
blockChain.SetGetEthLogsWithCustomQueryErr(fakes.FakeError) blockChain.SetGetEthLogsWithCustomQueryErr(fakes.FakeError)
fetcher := shared.NewFetcher(blockChain) fetcher := shared.NewFetcher(blockChain)
_, err := fetcher.FetchLogs([]string{}, [][]common.Hash{}, core.Header{}) _, err := fetcher.FetchLogs([]common.Address{}, []common.Hash{}, core.Header{})
Expect(err).To(HaveOccurred()) Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError)) Expect(err).To(MatchError(fakes.FakeError))

60
pkg/transformers/shared/repository.go
View File

@ -1,7 +1,10 @@
package shared package shared
import ( import (
"bytes"
"database/sql" "database/sql"
"database/sql/driver"
"fmt"
"github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
) )
@ -22,7 +25,8 @@ func MarkHeaderCheckedInTransaction(headerID int64, tx *sql.Tx, checkedHeadersCo
return err return err
} }
func MissingHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.DB, checkedHeadersColumn string) ([]core.Header, error) { // Treats a header as missing if it's not in the headers table, or not checked for some log type
func MissingHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.DB, notCheckedSQL string) ([]core.Header, error) {
var result []core.Header var result []core.Header
var query string var query string
var err error var err error
@ -30,14 +34,14 @@ func MissingHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.D
if endingBlockNumber == -1 { if endingBlockNumber == -1 {
query = `SELECT headers.id, headers.block_number, headers.hash FROM headers query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
LEFT JOIN checked_headers on headers.id = header_id LEFT JOIN checked_headers on headers.id = header_id
WHERE (header_id ISNULL OR ` + checkedHeadersColumn + ` IS FALSE) WHERE (header_id ISNULL OR ` + notCheckedSQL + `)
AND headers.block_number >= $1 AND headers.block_number >= $1
AND headers.eth_node_fingerprint = $2` AND headers.eth_node_fingerprint = $2`
err = db.Select(&result, query, startingBlockNumber, db.Node.ID) err = db.Select(&result, query, startingBlockNumber, db.Node.ID)
} else { } else {
query = `SELECT headers.id, headers.block_number, headers.hash FROM headers query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
LEFT JOIN checked_headers on headers.id = header_id LEFT JOIN checked_headers on headers.id = header_id
WHERE (header_id ISNULL OR ` + checkedHeadersColumn + ` IS FALSE) WHERE (header_id ISNULL OR ` + notCheckedSQL + `)
AND headers.block_number >= $1 AND headers.block_number >= $1
AND headers.block_number <= $2 AND headers.block_number <= $2
AND headers.eth_node_fingerprint = $3` AND headers.eth_node_fingerprint = $3`
@ -46,3 +50,53 @@ func MissingHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.D
return result, err return result, err
} }
func GetCheckedColumnNames(db *postgres.DB) ([]string, error) {
// Query returns `[]driver.Value`, nullable polymorphic interface
var queryResult []driver.Value
columnNamesQuery :=
`SELECT column_name FROM information_schema.columns
WHERE table_schema = 'public'
AND table_name = 'checked_headers'
AND column_name ~ '_checked';`
err := db.Select(&queryResult, columnNamesQuery)
if err != nil {
return []string{}, err
}
// Transform column names from `driver.Value` to strings
var columnNames []string
for _, result := range queryResult {
if columnName, ok := result.(string); ok {
columnNames = append(columnNames, columnName)
} else {
return []string{}, fmt.Errorf("incorrect value for checked_headers column name")
}
}
return columnNames, nil
}
// Builds a SQL string that checks if any column value is FALSE, given the column names.
// Defaults to FALSE when no columns are provided.
// Ex: ["columnA", "columnB"] => "NOT (columnA AND columnB)"
// [] => "FALSE"
func CreateNotCheckedSQL(boolColumns []string) string {
var result bytes.Buffer
if len(boolColumns) == 0 {
return "FALSE"
}
result.WriteString("NOT (")
// Loop excluding last column name
for _, column := range boolColumns[:len(boolColumns)-1] {
result.WriteString(fmt.Sprintf("%v AND ", column))
}
// No trailing "OR" for the last column name
result.WriteString(fmt.Sprintf("%v)", boolColumns[len(boolColumns)-1]))
return result.String()
}

170
pkg/transformers/shared/repository_utility_test.go Normal file
View File

@ -0,0 +1,170 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package shared_test
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/test_config"
"math/rand"
)
var _ = Describe("Repository utilities", func() {
Describe("MissingHeaders", func() {
var (
db *postgres.DB
headerRepository datastore.HeaderRepository
startingBlockNumber int64
endingBlockNumber int64
eventSpecificBlockNumber int64
blockNumbers []int64
headerIDs []int64
notCheckedSQL string
err error
)
BeforeEach(func() {
db = test_config.NewTestDB(test_config.NewTestNode())
test_config.CleanTestDB(db)
headerRepository = repositories.NewHeaderRepository(db)
columnNames, err := shared.GetCheckedColumnNames(db)
Expect(err).NotTo(HaveOccurred())
notCheckedSQL = shared.CreateNotCheckedSQL(columnNames)
startingBlockNumber = rand.Int63()
eventSpecificBlockNumber = startingBlockNumber + 1
endingBlockNumber = startingBlockNumber + 2
outOfRangeBlockNumber := endingBlockNumber + 1
blockNumbers = []int64{startingBlockNumber, eventSpecificBlockNumber, endingBlockNumber, outOfRangeBlockNumber}
headerIDs = []int64{}
for _, n := range blockNumbers {
headerID, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(n))
headerIDs = append(headerIDs, headerID)
Expect(err).NotTo(HaveOccurred())
}
})
It("only treats headers as checked if the event specific logs have been checked", func() {
_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
Expect(err).NotTo(HaveOccurred())
headers, err := shared.MissingHeaders(startingBlockNumber, endingBlockNumber, db, notCheckedSQL)
Expect(err).NotTo(HaveOccurred())
Expect(len(headers)).To(Equal(3))
Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber)))
Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber)))
Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber)))
})
It("only returns headers associated with the current node", func() {
dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
for _, n := range blockNumbers {
_, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n + 10))
Expect(err).NotTo(HaveOccurred())
}
Expect(err).NotTo(HaveOccurred())
nodeOneMissingHeaders, err := shared.MissingHeaders(startingBlockNumber, endingBlockNumber, db, notCheckedSQL)
Expect(err).NotTo(HaveOccurred())
Expect(len(nodeOneMissingHeaders)).To(Equal(3))
Expect(nodeOneMissingHeaders[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(eventSpecificBlockNumber), Equal(endingBlockNumber)))
Expect(nodeOneMissingHeaders[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(eventSpecificBlockNumber), Equal(endingBlockNumber)))
Expect(nodeOneMissingHeaders[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(startingBlockNumber), Equal(eventSpecificBlockNumber), Equal(endingBlockNumber)))
nodeTwoMissingHeaders, err := shared.MissingHeaders(startingBlockNumber, endingBlockNumber+10, dbTwo, notCheckedSQL)
Expect(err).NotTo(HaveOccurred())
Expect(len(nodeTwoMissingHeaders)).To(Equal(3))
Expect(nodeTwoMissingHeaders[0].BlockNumber).To(Or(Equal(startingBlockNumber+10), Equal(eventSpecificBlockNumber+10), Equal(endingBlockNumber+10)))
Expect(nodeTwoMissingHeaders[1].BlockNumber).To(Or(Equal(startingBlockNumber+10), Equal(eventSpecificBlockNumber+10), Equal(endingBlockNumber+10)))
})
})
Describe("GetCheckedColumnNames", func() {
It("gets the column names from checked_headers", func() {
db := test_config.NewTestDB(test_config.NewTestNode())
test_config.CleanTestDB(db)
expectedColumnNames := getExpectedColumnNames()
actualColumnNames, err := shared.GetCheckedColumnNames(db)
Expect(err).NotTo(HaveOccurred())
Expect(actualColumnNames).To(Equal(expectedColumnNames))
})
})
Describe("CreateNotCheckedSQL", func() {
It("generates a correct SQL string for one column", func() {
columns := []string{"columnA"}
expected := "NOT (columnA)"
actual := shared.CreateNotCheckedSQL(columns)
Expect(actual).To(Equal(expected))
})
It("generates a correct SQL string for several columns", func() {
columns := []string{"columnA", "columnB"}
expected := "NOT (columnA AND columnB)"
actual := shared.CreateNotCheckedSQL(columns)
Expect(actual).To(Equal(expected))
})
It("defaults to FALSE when there are no columns", func() {
expected := "FALSE"
actual := shared.CreateNotCheckedSQL([]string{})
Expect(actual).To(Equal(expected))
})
})
})
func getExpectedColumnNames() []string {
return []string{
"price_feeds_checked",
"flip_kick_checked",
"frob_checked",
"tend_checked",
"bite_checked",
"dent_checked",
"pit_file_debt_ceiling_checked",
"pit_file_ilk_checked",
"vat_init_checked",
"drip_file_ilk_checked",
"drip_file_repo_checked",
"drip_file_vow_checked",
"deal_checked",
"drip_drip_checked",
"cat_file_chop_lump_checked",
"cat_file_flip_checked",
"cat_file_pit_vow_checked",
"flop_kick_checked",
"vat_move_checked",
"vat_fold_checked",
"vat_heal_checked",
"vat_toll_checked",
"vat_tune_checked",
"vat_grab_checked",
"vat_flux_checked",
"vat_slip_checked",
"vow_flog_checked",
"flap_kick_checked",
}
}

15
pkg/transformers/shared/transformer.go
View File

@ -16,16 +16,18 @@ package shared
import ( import (
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
) )
type Transformer interface { type Transformer interface {
Execute() error Execute(logs []types.Log, header core.Header) error
GetConfig() TransformerConfig
} }
type TransformerInitializer func(db *postgres.DB, blockChain core.BlockChain) Transformer type TransformerInitializer func(db *postgres.DB) Transformer
type TransformerConfig struct { type TransformerConfig struct {
TransformerName string TransformerName string
@ -45,3 +47,10 @@ func HexToString(byteString string) string {
value := common.HexToHash(byteString) value := common.HexToHash(byteString)
return value.Big().String() return value.Big().String()
} }
func HexStringsToAddresses(strings []string) (addresses []common.Address) {
for _, hexString := range strings {
addresses = append(addresses, common.HexToAddress(hexString))
}
return
}

5
pkg/transformers/tend/repository.go
View File

@ -16,7 +16,6 @@ package tend
import ( import (
"fmt" "fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -68,10 +67,6 @@ func (repository TendRepository) MarkHeaderChecked(headerId int64) error {
return shared.MarkHeaderChecked(headerId, repository.db, constants.TendChecked) return shared.MarkHeaderChecked(headerId, repository.db, constants.TendChecked)
} }
func (repository TendRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.TendChecked)
}
func (repository *TendRepository) SetDB(db *postgres.DB) { func (repository *TendRepository) SetDB(db *postgres.DB) {
repository.db = db repository.db = db
} }

9
pkg/transformers/tend/repository_test.go
View File

@ -94,13 +94,4 @@ var _ = Describe("TendRepository", func() {
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
}) })
Describe("MissingHeaders", func() {
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
Repository: &tendRepository,
RepositoryTwo: &tend.TendRepository{},
}
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
})
}) })

8
pkg/transformers/test_data/mocks/log_fetcher.go
View File

@ -21,18 +21,20 @@ import (
) )
type MockLogFetcher struct { type MockLogFetcher struct {
FetchedContractAddresses [][]string FetchedContractAddresses [][]common.Address
FetchedTopics [][]common.Hash FetchedTopics [][]common.Hash
FetchedBlocks []int64 FetchedBlocks []int64
fetcherError error fetcherError error
FetchedLogs []types.Log FetchedLogs []types.Log
SetBcCalled bool SetBcCalled bool
FetchLogsCalled bool
} }
func (mlf *MockLogFetcher) FetchLogs(contractAddresses []string, topics [][]common.Hash, header core.Header) ([]types.Log, error) { func (mlf *MockLogFetcher) FetchLogs(contractAddresses []common.Address, topics []common.Hash, header core.Header) ([]types.Log, error) {
mlf.FetchedContractAddresses = append(mlf.FetchedContractAddresses, contractAddresses) mlf.FetchedContractAddresses = append(mlf.FetchedContractAddresses, contractAddresses)
mlf.FetchedTopics = topics mlf.FetchedTopics = [][]common.Hash{topics}
mlf.FetchedBlocks = append(mlf.FetchedBlocks, header.BlockNumber) mlf.FetchedBlocks = append(mlf.FetchedBlocks, header.BlockNumber)
mlf.FetchLogsCalled = true
return mlf.FetchedLogs, mlf.fetcherError return mlf.FetchedLogs, mlf.fetcherError
} }

53
pkg/transformers/test_data/mocks/mock_watcher_repository.go Normal file
View File

@ -0,0 +1,53 @@
package mocks
import (
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
type MockWatcherRepository struct {
ReturnCheckedColumnNames []string
GetCheckedColumnNamesError error
GetCheckedColumnNamesCalled bool
ReturnNotCheckedSQL string
CreateNotCheckedSQLCalled bool
ReturnMissingHeaders []core.Header
MissingHeadersError error
MissingHeadersCalled bool
}
func (repository *MockWatcherRepository) GetCheckedColumnNames(db *postgres.DB) ([]string, error) {
repository.GetCheckedColumnNamesCalled = true
if repository.GetCheckedColumnNamesError != nil {
return []string{}, repository.GetCheckedColumnNamesError
}
return repository.ReturnCheckedColumnNames, nil
}
func (repository *MockWatcherRepository) SetCheckedColumnNames(checkedColumnNames []string) {
repository.ReturnCheckedColumnNames = checkedColumnNames
}
func (repository *MockWatcherRepository) CreateNotCheckedSQL(boolColumns []string) string {
repository.CreateNotCheckedSQLCalled = true
return repository.ReturnNotCheckedSQL
}
func (repository *MockWatcherRepository) SetNotCheckedSQL(notCheckedSql string) {
repository.ReturnNotCheckedSQL = notCheckedSql
}
func (repository *MockWatcherRepository) MissingHeaders(startingBlockNumber int64, endingBlockNumber int64, db *postgres.DB, notCheckedSQL string) ([]core.Header, error) {
if repository.MissingHeadersError != nil {
return []core.Header{}, repository.MissingHeadersError
}
repository.MissingHeadersCalled = true
return repository.ReturnMissingHeaders, nil
}
func (repository *MockWatcherRepository) SetMissingHeaders(headers []core.Header) {
repository.ReturnMissingHeaders = headers
}

17
pkg/transformers/test_data/mocks/repository.go
View File

@ -3,7 +3,6 @@ package mocks
import ( import (
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
) )
@ -12,8 +11,6 @@ type MockRepository struct {
markHeaderCheckedError error markHeaderCheckedError error
MarkHeaderCheckedPassedHeaderIDs []int64 MarkHeaderCheckedPassedHeaderIDs []int64
CreatedHeaderIds []int64 CreatedHeaderIds []int64
missingHeaders []core.Header
missingHeadersError error
PassedStartingBlockNumber int64 PassedStartingBlockNumber int64
PassedEndingBlockNumber int64 PassedEndingBlockNumber int64
PassedHeaderID int64 PassedHeaderID int64
@ -36,24 +33,10 @@ func (repository *MockRepository) MarkHeaderChecked(headerID int64) error {
return repository.markHeaderCheckedError return repository.markHeaderCheckedError
} }
func (repository *MockRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
repository.PassedStartingBlockNumber = startingBlockNumber
repository.PassedEndingBlockNumber = endingBlockNumber
return repository.missingHeaders, repository.missingHeadersError
}
func (repository *MockRepository) SetDB(db *postgres.DB) { func (repository *MockRepository) SetDB(db *postgres.DB) {
repository.SetDbCalled = true repository.SetDbCalled = true
} }
func (repository *MockRepository) SetMissingHeadersError(e error) {
repository.missingHeadersError = e
}
func (repository *MockRepository) SetMissingHeaders(headers []core.Header) {
repository.missingHeaders = headers
}
func (repository *MockRepository) SetMarkHeaderCheckedError(e error) { func (repository *MockRepository) SetMarkHeaderCheckedError(e error) {
repository.markHeaderCheckedError = e repository.markHeaderCheckedError = e
} }

44
pkg/transformers/test_data/mocks/transformer.go Normal file
View File

@ -0,0 +1,44 @@
package mocks
import (
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
)
type MockTransformer struct {
ExecuteWasCalled bool
ExecuteError error
PassedLogs []types.Log
PassedHeader core.Header
config shared.TransformerConfig
}
func (mh *MockTransformer) Execute(logs []types.Log, header core.Header) error {
if mh.ExecuteError != nil {
return mh.ExecuteError
}
mh.ExecuteWasCalled = true
mh.PassedLogs = logs
mh.PassedHeader = header
return nil
}
func (mh *MockTransformer) GetConfig() shared.TransformerConfig {
return mh.config
}
func (mh *MockTransformer) SetTransformerConfig(config shared.TransformerConfig) {
mh.config = config
}
func (mh *MockTransformer) FakeTransformerInitializer(db *postgres.DB) shared.Transformer {
return mh
}
var FakeTransformerConfig = shared.TransformerConfig{
TransformerName: "FakeTransformer",
ContractAddresses: []string{"FakeAddress"},
Topic: "FakeTopic",
}

89
pkg/transformers/test_data/shared_behaviors/repository_behaviors.go
View File

@ -17,7 +17,6 @@ package shared_behaviors
import ( import (
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore" "github.com/vulcanize/vulcanizedb/pkg/datastore"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
@ -25,7 +24,6 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
"github.com/vulcanize/vulcanizedb/test_config" "github.com/vulcanize/vulcanizedb/test_config"
"math/rand"
) )
var ( var (
@ -61,7 +59,7 @@ func SharedRepositoryCreateBehaviors(inputs *CreateBehaviorInputs) {
var logEventModel = inputs.TestModel var logEventModel = inputs.TestModel
BeforeEach(func() { BeforeEach(func() {
headerRepository = GetHeaderRepository() headerRepository = getHeaderRepository()
headerID, err = headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) headerID, err = headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
}) })
@ -132,94 +130,13 @@ func SharedRepositoryCreateBehaviors(inputs *CreateBehaviorInputs) {
}) })
} }
func SharedRepositoryMissingHeadersBehaviors(inputs *MissingHeadersBehaviorInputs) {
Describe("MissingHeaders", func() {
var (
repository = inputs.Repository
startingBlockNumber int64
endingBlockNumber int64
eventSpecificBlockNumber int64
blockNumbers []int64
headerIDs []int64
)
BeforeEach(func() {
headerRepository = GetHeaderRepository()
startingBlockNumber = rand.Int63()
eventSpecificBlockNumber = startingBlockNumber + 1
endingBlockNumber = startingBlockNumber + 2
outOfRangeBlockNumber := endingBlockNumber + 1
blockNumbers = []int64{startingBlockNumber, eventSpecificBlockNumber, endingBlockNumber, outOfRangeBlockNumber}
headerIDs = []int64{}
for _, n := range blockNumbers {
headerID, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(n))
headerIDs = append(headerIDs, headerID)
Expect(err).NotTo(HaveOccurred())
}
})
It("returns headers that haven't been checked", func() {
err := repository.MarkHeaderChecked(headerIDs[1])
Expect(err).NotTo(HaveOccurred())
headers, err := repository.MissingHeaders(startingBlockNumber, endingBlockNumber)
Expect(err).NotTo(HaveOccurred())
Expect(len(headers)).To(Equal(2))
Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
})
It("only treats headers as checked if the event specific logs have been checked", func() {
_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
Expect(err).NotTo(HaveOccurred())
headers, err := repository.MissingHeaders(startingBlockNumber, endingBlockNumber)
Expect(err).NotTo(HaveOccurred())
Expect(len(headers)).To(Equal(3))
Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber)))
Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber)))
Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber)))
})
It("only returns headers associated with the current node", func() {
dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
for _, n := range blockNumbers {
_, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n))
Expect(err).NotTo(HaveOccurred())
}
repositoryTwo := inputs.RepositoryTwo
repositoryTwo.SetDB(dbTwo)
err := repository.MarkHeaderChecked(headerIDs[0])
Expect(err).NotTo(HaveOccurred())
nodeOneMissingHeaders, err := repository.MissingHeaders(startingBlockNumber, endingBlockNumber)
Expect(err).NotTo(HaveOccurred())
Expect(len(nodeOneMissingHeaders)).To(Equal(2))
Expect(nodeOneMissingHeaders[0].BlockNumber).To(Or(Equal(eventSpecificBlockNumber), Equal(endingBlockNumber)))
Expect(nodeOneMissingHeaders[1].BlockNumber).To(Or(Equal(eventSpecificBlockNumber), Equal(endingBlockNumber)))
nodeTwoMissingHeaders, err := repositoryTwo.MissingHeaders(startingBlockNumber, endingBlockNumber)
Expect(err).NotTo(HaveOccurred())
Expect(len(nodeTwoMissingHeaders)).To(Equal(3))
Expect(nodeTwoMissingHeaders[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(eventSpecificBlockNumber), Equal(endingBlockNumber)))
Expect(nodeTwoMissingHeaders[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(eventSpecificBlockNumber), Equal(endingBlockNumber)))
Expect(nodeTwoMissingHeaders[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(eventSpecificBlockNumber), Equal(endingBlockNumber)))
})
})
}
func SharedRepositoryMarkHeaderCheckedBehaviors(inputs *MarkedHeaderCheckedBehaviorInputs) { func SharedRepositoryMarkHeaderCheckedBehaviors(inputs *MarkedHeaderCheckedBehaviorInputs) {
var repository = inputs.Repository var repository = inputs.Repository
var checkedHeaderColumn = inputs.CheckedHeaderColumnName var checkedHeaderColumn = inputs.CheckedHeaderColumnName
Describe("MarkHeaderChecked", func() { Describe("MarkHeaderChecked", func() {
BeforeEach(func() { BeforeEach(func() {
headerRepository = GetHeaderRepository() headerRepository = getHeaderRepository()
headerId, err = headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) headerId, err = headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
}) })
@ -257,7 +174,7 @@ func SharedRepositoryMarkHeaderCheckedBehaviors(inputs *MarkedHeaderCheckedBehav
}) })
} }
func GetHeaderRepository() repositories.HeaderRepository { func getHeaderRepository() repositories.HeaderRepository {
db = test_config.NewTestDB(test_config.NewTestNode()) db = test_config.NewTestDB(test_config.NewTestNode())
test_config.CleanTestDB(db) test_config.CleanTestDB(db)

298
pkg/transformers/transformers.go
View File

@ -47,233 +47,223 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/transformers/vow_flog" "github.com/vulcanize/vulcanizedb/pkg/transformers/vow_flog"
) )
// Custom event transformers
var ( var (
BiteTransformerInitializer = factories.Transformer{ BiteTransformer = factories.Transformer{
Config: bite.BiteConfig, Config: bite.BiteConfig,
Converter: &bite.BiteConverter{}, Converter: &bite.BiteConverter{},
Repository: &bite.BiteRepository{}, Repository: &bite.BiteRepository{},
Fetcher: &shared.Fetcher{}, }
}.NewTransformer
CatFileChopLumpTransformerInitializer = factories.LogNoteTransformer{ FlapKickTransformer = factories.Transformer{
Config: chop_lump.CatFileChopLumpConfig,
Converter: &chop_lump.CatFileChopLumpConverter{},
Repository: &chop_lump.CatFileChopLumpRepository{},
Fetcher: &shared.Fetcher{},
}.NewLogNoteTransformer
CatFileFlipTransformerInitializer = factories.LogNoteTransformer{
Config: flip.CatFileFlipConfig,
Converter: &flip.CatFileFlipConverter{},
Repository: &flip.CatFileFlipRepository{},
Fetcher: &shared.Fetcher{},
}.NewLogNoteTransformer
CatFilePitVowTransformerInitializer = factories.LogNoteTransformer{
Config: pit_vow.CatFilePitVowConfig,
Converter: &pit_vow.CatFilePitVowConverter{},
Repository: &pit_vow.CatFilePitVowRepository{},
Fetcher: &shared.Fetcher{},
}.NewLogNoteTransformer
DealTransformerInitializer = factories.LogNoteTransformer{
Config: deal.DealConfig,
Converter: &deal.DealConverter{},
Repository: &deal.DealRepository{},
Fetcher: &shared.Fetcher{},
}.NewLogNoteTransformer
DentTransformerInitializer = factories.LogNoteTransformer{
Config: dent.DentConfig,
Converter: &dent.DentConverter{},
Repository: &dent.DentRepository{},
Fetcher: &shared.Fetcher{},
}.NewLogNoteTransformer
DripDripTransformerInitializer = factories.LogNoteTransformer{
Config: drip_drip.DripDripConfig,
Converter: &drip_drip.DripDripConverter{},
Repository: &drip_drip.DripDripRepository{},
Fetcher: &shared.Fetcher{},
}.NewLogNoteTransformer
DripFileIlkTransformerInitializer = factories.LogNoteTransformer{
Config: ilk2.DripFileIlkConfig,
Converter: &ilk2.DripFileIlkConverter{},
Repository: &ilk2.DripFileIlkRepository{},
Fetcher: &shared.Fetcher{},
}.NewLogNoteTransformer
DripFileRepoTransformerInitializer = factories.LogNoteTransformer{
Config: repo.DripFileRepoConfig,
Converter: &repo.DripFileRepoConverter{},
Repository: &repo.DripFileRepoRepository{},
Fetcher: &shared.Fetcher{},
}.NewLogNoteTransformer
DripFileVowTransfromerInitializer = factories.LogNoteTransformer{
Config: vow.DripFileVowConfig,
Converter: &vow.DripFileVowConverter{},
Repository: &vow.DripFileVowRepository{},
Fetcher: &shared.Fetcher{},
}.NewLogNoteTransformer
FlapKickTransformerInitializer = factories.Transformer{
Config: flap_kick.FlapKickConfig, Config: flap_kick.FlapKickConfig,
Converter: &flap_kick.FlapKickConverter{}, Converter: &flap_kick.FlapKickConverter{},
Repository: &flap_kick.FlapKickRepository{}, Repository: &flap_kick.FlapKickRepository{},
Fetcher: &shared.Fetcher{}, }
}.NewTransformer
FlipKickTransformerInitializer = factories.Transformer{ FlipKickTransformer = factories.Transformer{
Config: flip_kick.FlipKickConfig, Config: flip_kick.FlipKickConfig,
Converter: &flip_kick.FlipKickConverter{}, Converter: &flip_kick.FlipKickConverter{},
Repository: &flip_kick.FlipKickRepository{}, Repository: &flip_kick.FlipKickRepository{},
Fetcher: &shared.Fetcher{}, }
}.NewTransformer
FlogTransformerInitializer = factories.LogNoteTransformer{ FrobTransformer = factories.Transformer{
Config: vow_flog.VowFlogConfig,
Converter: &vow_flog.VowFlogConverter{},
Repository: &vow_flog.VowFlogRepository{},
Fetcher: &shared.Fetcher{},
}.NewLogNoteTransformer
FrobTransformerInitializer = factories.Transformer{
Config: frob.FrobConfig, Config: frob.FrobConfig,
Converter: &frob.FrobConverter{}, Converter: &frob.FrobConverter{},
Repository: &frob.FrobRepository{}, Repository: &frob.FrobRepository{},
Fetcher: &shared.Fetcher{}, }
}.NewTransformer
FlopKickTransformerInitializer = factories.Transformer{ FlopKickTransformer = factories.Transformer{
Config: flop_kick.Config, Config: flop_kick.Config,
Converter: &flop_kick.FlopKickConverter{}, Converter: &flop_kick.FlopKickConverter{},
Repository: &flop_kick.FlopKickRepository{}, Repository: &flop_kick.FlopKickRepository{},
Fetcher: &shared.Fetcher{}, }
}.NewTransformer
PitFileDebtCeilingTransformerInitializer = factories.LogNoteTransformer{ customEventTransformers = []factories.Transformer{
BiteTransformer,
FlapKickTransformer,
FlipKickTransformer,
FrobTransformer,
FlopKickTransformer,
}
)
// LogNote transformers
var (
CatFileChopLumpTransformer = factories.LogNoteTransformer{
Config: chop_lump.CatFileChopLumpConfig,
Converter: &chop_lump.CatFileChopLumpConverter{},
Repository: &chop_lump.CatFileChopLumpRepository{},
}
CatFileFlipTransformer = factories.LogNoteTransformer{
Config: flip.CatFileFlipConfig,
Converter: &flip.CatFileFlipConverter{},
Repository: &flip.CatFileFlipRepository{},
}
CatFilePitVowTransformer = factories.LogNoteTransformer{
Config: pit_vow.CatFilePitVowConfig,
Converter: &pit_vow.CatFilePitVowConverter{},
Repository: &pit_vow.CatFilePitVowRepository{},
}
DealTransformer = factories.LogNoteTransformer{
Config: deal.DealConfig,
Converter: &deal.DealConverter{},
Repository: &deal.DealRepository{},
}
DentTransformer = factories.LogNoteTransformer{
Config: dent.DentConfig,
Converter: &dent.DentConverter{},
Repository: &dent.DentRepository{},
}
DripDripTransformer = factories.LogNoteTransformer{
Config: drip_drip.DripDripConfig,
Converter: &drip_drip.DripDripConverter{},
Repository: &drip_drip.DripDripRepository{},
}
DripFileIlkTransformer = factories.LogNoteTransformer{
Config: ilk2.DripFileIlkConfig,
Converter: &ilk2.DripFileIlkConverter{},
Repository: &ilk2.DripFileIlkRepository{},
}
DripFileRepoTransformer = factories.LogNoteTransformer{
Config: repo.DripFileRepoConfig,
Converter: &repo.DripFileRepoConverter{},
Repository: &repo.DripFileRepoRepository{},
}
DripFileVowTransfromer = factories.LogNoteTransformer{
Config: vow.DripFileVowConfig,
Converter: &vow.DripFileVowConverter{},
Repository: &vow.DripFileVowRepository{},
}
FlogTransformer = factories.LogNoteTransformer{
Config: vow_flog.VowFlogConfig,
Converter: &vow_flog.VowFlogConverter{},
Repository: &vow_flog.VowFlogRepository{},
}
PitFileDebtCeilingTransformer = factories.LogNoteTransformer{
Config: debt_ceiling.DebtCeilingFileConfig, Config: debt_ceiling.DebtCeilingFileConfig,
Converter: &debt_ceiling.PitFileDebtCeilingConverter{}, Converter: &debt_ceiling.PitFileDebtCeilingConverter{},
Repository: &debt_ceiling.PitFileDebtCeilingRepository{}, Repository: &debt_ceiling.PitFileDebtCeilingRepository{},
Fetcher: &shared.Fetcher{}, }
}.NewLogNoteTransformer
PitFileIlkTransformerInitializer = factories.LogNoteTransformer{ PitFileIlkTransformer = factories.LogNoteTransformer{
Config: ilk.IlkFileConfig, Config: ilk.IlkFileConfig,
Converter: &ilk.PitFileIlkConverter{}, Converter: &ilk.PitFileIlkConverter{},
Repository: &ilk.PitFileIlkRepository{}, Repository: &ilk.PitFileIlkRepository{},
Fetcher: &shared.Fetcher{}, }
}.NewLogNoteTransformer
PriceFeedTransformerInitializer = factories.LogNoteTransformer{ PriceFeedTransformer = factories.LogNoteTransformer{
Config: price_feeds.PriceFeedConfig, Config: price_feeds.PriceFeedConfig,
Converter: &price_feeds.PriceFeedConverter{}, Converter: &price_feeds.PriceFeedConverter{},
Repository: &price_feeds.PriceFeedRepository{}, Repository: &price_feeds.PriceFeedRepository{},
Fetcher: &shared.Fetcher{}, }
}.NewLogNoteTransformer
TendTransformerInitializer = factories.LogNoteTransformer{ TendTransformer = factories.LogNoteTransformer{
Config: tend.TendConfig, Config: tend.TendConfig,
Converter: &tend.TendConverter{}, Converter: &tend.TendConverter{},
Repository: &tend.TendRepository{}, Repository: &tend.TendRepository{},
Fetcher: &shared.Fetcher{}, }
}.NewLogNoteTransformer
VatInitTransformerInitializer = factories.LogNoteTransformer{ VatInitTransformer = factories.LogNoteTransformer{
Config: vat_init.VatInitConfig, Config: vat_init.VatInitConfig,
Converter: &vat_init.VatInitConverter{}, Converter: &vat_init.VatInitConverter{},
Repository: &vat_init.VatInitRepository{}, Repository: &vat_init.VatInitRepository{},
Fetcher: &shared.Fetcher{}, }
}.NewLogNoteTransformer
VatGrabTransformerInitializer = factories.LogNoteTransformer{ VatGrabTransformer = factories.LogNoteTransformer{
Config: vat_grab.VatGrabConfig, Config: vat_grab.VatGrabConfig,
Converter: &vat_grab.VatGrabConverter{}, Converter: &vat_grab.VatGrabConverter{},
Repository: &vat_grab.VatGrabRepository{}, Repository: &vat_grab.VatGrabRepository{},
Fetcher: &shared.Fetcher{}, }
}.NewLogNoteTransformer
VatFoldTransformerInitializer = factories.LogNoteTransformer{ VatFoldTransformer = factories.LogNoteTransformer{
Config: vat_fold.VatFoldConfig, Config: vat_fold.VatFoldConfig,
Converter: &vat_fold.VatFoldConverter{}, Converter: &vat_fold.VatFoldConverter{},
Repository: &vat_fold.VatFoldRepository{}, Repository: &vat_fold.VatFoldRepository{},
Fetcher: &shared.Fetcher{}, }
}.NewLogNoteTransformer
VatHealTransformerInitializer = factories.LogNoteTransformer{ VatHealTransformer = factories.LogNoteTransformer{
Config: vat_heal.VatHealConfig, Config: vat_heal.VatHealConfig,
Converter: &vat_heal.VatHealConverter{}, Converter: &vat_heal.VatHealConverter{},
Repository: &vat_heal.VatHealRepository{}, Repository: &vat_heal.VatHealRepository{},
Fetcher: &shared.Fetcher{}, }
}.NewLogNoteTransformer
VatMoveTransformerInitializer = factories.LogNoteTransformer{ VatMoveTransformer = factories.LogNoteTransformer{
Config: vat_move.VatMoveConfig, Config: vat_move.VatMoveConfig,
Converter: &vat_move.VatMoveConverter{}, Converter: &vat_move.VatMoveConverter{},
Repository: &vat_move.VatMoveRepository{}, Repository: &vat_move.VatMoveRepository{},
Fetcher: &shared.Fetcher{}, }
}.NewLogNoteTransformer
VatSlipTransformerInitializer = factories.LogNoteTransformer{ VatSlipTransformer = factories.LogNoteTransformer{
Config: vat_slip.VatSlipConfig, Config: vat_slip.VatSlipConfig,
Converter: &vat_slip.VatSlipConverter{}, Converter: &vat_slip.VatSlipConverter{},
Repository: &vat_slip.VatSlipRepository{}, Repository: &vat_slip.VatSlipRepository{},
Fetcher: &shared.Fetcher{}, }
}.NewLogNoteTransformer
VatTollTransformerInitializer = factories.LogNoteTransformer{ VatTollTransformer = factories.LogNoteTransformer{
Config: vat_toll.VatTollConfig, Config: vat_toll.VatTollConfig,
Converter: &vat_toll.VatTollConverter{}, Converter: &vat_toll.VatTollConverter{},
Repository: &vat_toll.VatTollRepository{}, Repository: &vat_toll.VatTollRepository{},
Fetcher: &shared.Fetcher{}, }
}.NewLogNoteTransformer
VatTuneTransformerInitializer = factories.LogNoteTransformer{ VatTuneTransformer = factories.LogNoteTransformer{
Config: vat_tune.VatTuneConfig, Config: vat_tune.VatTuneConfig,
Converter: &vat_tune.VatTuneConverter{}, Converter: &vat_tune.VatTuneConverter{},
Repository: &vat_tune.VatTuneRepository{}, Repository: &vat_tune.VatTuneRepository{},
Fetcher: &shared.Fetcher{}, }
}.NewLogNoteTransformer
VatFluxTransformerInitializer = factories.LogNoteTransformer{ VatFluxTransformer = factories.LogNoteTransformer{
Config: vat_flux.VatFluxConfig, Config: vat_flux.VatFluxConfig,
Converter: &vat_flux.VatFluxConverter{}, Converter: &vat_flux.VatFluxConverter{},
Repository: &vat_flux.VatFluxRepository{}, Repository: &vat_flux.VatFluxRepository{},
Fetcher: &shared.Fetcher{}, }
}.NewLogNoteTransformer
logNoteTransformers = []factories.LogNoteTransformer{
CatFileChopLumpTransformer,
CatFileFlipTransformer,
CatFilePitVowTransformer,
DealTransformer,
DentTransformer,
DripDripTransformer,
DripFileIlkTransformer,
DripFileRepoTransformer,
DripFileVowTransfromer,
FlogTransformer,
PitFileDebtCeilingTransformer,
PitFileIlkTransformer,
PriceFeedTransformer,
TendTransformer,
VatInitTransformer,
VatGrabTransformer,
VatFoldTransformer,
VatHealTransformer,
VatMoveTransformer,
VatSlipTransformer,
VatTollTransformer,
VatTuneTransformer,
VatFluxTransformer,
}
) )
func TransformerInitializers() []shared.TransformerInitializer { // `TransformerInitializers` returns a list of functions, that given a db pointer
return []shared.TransformerInitializer{ // will return a `shared.Transformer`
BiteTransformerInitializer, func TransformerInitializers() (initializers []shared.TransformerInitializer) {
CatFileChopLumpTransformerInitializer, for _, transformer := range logNoteTransformers {
CatFileFlipTransformerInitializer, initializers = append(initializers, transformer.NewLogNoteTransformer)
CatFilePitVowTransformerInitializer,
DealTransformerInitializer,
DentTransformerInitializer,
DripDripTransformerInitializer,
DripFileIlkTransformerInitializer,
DripFileVowTransfromerInitializer,
DripFileRepoTransformerInitializer,
FlapKickTransformerInitializer,
FlipKickTransformerInitializer,
FlogTransformerInitializer,
FlopKickTransformerInitializer,
FrobTransformerInitializer,
PitFileDebtCeilingTransformerInitializer,
PitFileIlkTransformerInitializer,
PriceFeedTransformerInitializer,
TendTransformerInitializer,
VatGrabTransformerInitializer,
VatInitTransformerInitializer,
VatMoveTransformerInitializer,
VatHealTransformerInitializer,
VatFoldTransformerInitializer,
VatSlipTransformerInitializer,
VatTollTransformerInitializer,
VatTuneTransformerInitializer,
VatFluxTransformerInitializer,
} }
for _, transformer := range customEventTransformers {
initializers = append(initializers, transformer.NewTransformer)
}
return
} }

Some files were not shown because too many files have changed in this diff Show More