Merge branch 'staging' into batch-historical-headers
This commit is contained in:
Takayuki Goto
GitHub
commit
ac430ddf54
@ -17,7 +17,6 @@ package cmd
|
||||
import (
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/libraries/shared"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers"
|
||||
@ -46,13 +45,13 @@ func backfillMakerLogs() {
|
||||
log.Fatal("Failed to initialize database.")
|
||||
}
|
||||
|
||||
watcher := shared.Watcher{
|
||||
DB: *db,
|
||||
Blockchain: blockChain,
|
||||
}
|
||||
watcher := shared.NewWatcher(db, blockChain)
|
||||
|
||||
watcher.AddTransformers(transformers.TransformerInitializers())
|
||||
watcher.Execute()
|
||||
err = watcher.Execute()
|
||||
if err != nil {
|
||||
// TODO Handle watcher error in backfillMakerLogs
|
||||
}
|
||||
}
|
||||
|
||||
func init() {
|
||||
|
||||
@ -58,70 +58,66 @@ func syncMakerLogs() {
|
||||
log.Fatal("Failed to initialize database.")
|
||||
}
|
||||
|
||||
watcher := shared.Watcher{
|
||||
DB: *db,
|
||||
Blockchain: blockChain,
|
||||
}
|
||||
initializers := getTransformerInitializers(transformerNames)
|
||||
|
||||
transformerInititalizers := getTransformerInititalizers(transformerNames)
|
||||
watcher.AddTransformers(transformerInititalizers)
|
||||
watcher := shared.NewWatcher(db, blockChain)
|
||||
watcher.AddTransformers(initializers)
|
||||
|
||||
for range ticker.C {
|
||||
watcher.Execute()
|
||||
err = watcher.Execute()
|
||||
if err != nil {
|
||||
// TODO Handle watcher errors in ContinuousLogSync
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func getTransformerInititalizers(transformerNames []string) []shared2.TransformerInitializer {
|
||||
transformerInitializerMap := buildTransformerInitializerMap()
|
||||
var transformerInitializers []shared2.TransformerInitializer
|
||||
func getTransformerInitializers(transformerNames []string) []shared2.TransformerInitializer {
|
||||
var initializers []shared2.TransformerInitializer
|
||||
|
||||
if transformerNames[0] == "all" {
|
||||
for _, v := range transformerInitializerMap {
|
||||
transformerInitializers = append(transformerInitializers, v)
|
||||
}
|
||||
initializers = transformers.TransformerInitializers()
|
||||
} else {
|
||||
initializerMap := buildTransformerInitializerMap()
|
||||
for _, transformerName := range transformerNames {
|
||||
initializer := transformerInitializerMap[transformerName]
|
||||
transformerInitializers = append(transformerInitializers, initializer)
|
||||
initializers = append(initializers, initializerMap[transformerName])
|
||||
}
|
||||
}
|
||||
|
||||
return transformerInitializers
|
||||
return initializers
|
||||
}
|
||||
|
||||
func buildTransformerInitializerMap() map[string]shared2.TransformerInitializer {
|
||||
transformerInitializerMap := make(map[string]shared2.TransformerInitializer)
|
||||
initializerMap := make(map[string]shared2.TransformerInitializer)
|
||||
|
||||
transformerInitializerMap[constants.BiteLabel] = transformers.BiteTransformerInitializer
|
||||
transformerInitializerMap[constants.CatFileChopLumpLabel] = transformers.CatFileChopLumpTransformerInitializer
|
||||
transformerInitializerMap[constants.CatFileFlipLabel] = transformers.CatFileFlipTransformerInitializer
|
||||
transformerInitializerMap[constants.CatFilePitVowLabel] = transformers.CatFilePitVowTransformerInitializer
|
||||
transformerInitializerMap[constants.DealLabel] = transformers.DealTransformerInitializer
|
||||
transformerInitializerMap[constants.DentLabel] = transformers.DentTransformerInitializer
|
||||
transformerInitializerMap[constants.DripDripLabel] = transformers.DripDripTransformerInitializer
|
||||
transformerInitializerMap[constants.DripFileIlkLabel] = transformers.DripFileIlkTransformerInitializer
|
||||
transformerInitializerMap[constants.DripFileRepoLabel] = transformers.DripFileRepoTransformerInitializer
|
||||
transformerInitializerMap[constants.DripFileVowLabel] = transformers.DripFileVowTransfromerInitializer
|
||||
transformerInitializerMap[constants.FlapKickLabel] = transformers.FlapKickTransformerInitializer
|
||||
transformerInitializerMap[constants.FlipKickLabel] = transformers.FlipKickTransformerInitializer
|
||||
transformerInitializerMap[constants.FlopKickLabel] = transformers.FlopKickTransformerInitializer
|
||||
transformerInitializerMap[constants.FrobLabel] = transformers.FrobTransformerInitializer
|
||||
transformerInitializerMap[constants.PitFileDebtCeilingLabel] = transformers.PitFileDebtCeilingTransformerInitializer
|
||||
transformerInitializerMap[constants.PitFileIlkLabel] = transformers.PitFileIlkTransformerInitializer
|
||||
transformerInitializerMap[constants.PriceFeedLabel] = transformers.PriceFeedTransformerInitializer
|
||||
transformerInitializerMap[constants.TendLabel] = transformers.TendTransformerInitializer
|
||||
transformerInitializerMap[constants.VatFluxLabel] = transformers.VatFluxTransformerInitializer
|
||||
transformerInitializerMap[constants.VatFoldLabel] = transformers.VatFoldTransformerInitializer
|
||||
transformerInitializerMap[constants.VatGrabLabel] = transformers.VatGrabTransformerInitializer
|
||||
transformerInitializerMap[constants.VatHealLabel] = transformers.VatHealTransformerInitializer
|
||||
transformerInitializerMap[constants.VatInitLabel] = transformers.VatInitTransformerInitializer
|
||||
transformerInitializerMap[constants.VatMoveLabel] = transformers.VatMoveTransformerInitializer
|
||||
transformerInitializerMap[constants.VatSlipLabel] = transformers.VatSlipTransformerInitializer
|
||||
transformerInitializerMap[constants.VatTollLabel] = transformers.VatTollTransformerInitializer
|
||||
transformerInitializerMap[constants.VatTuneLabel] = transformers.VatTuneTransformerInitializer
|
||||
transformerInitializerMap[constants.VowFlogLabel] = transformers.FlogTransformerInitializer
|
||||
initializerMap[constants.BiteLabel] = transformers.BiteTransformer.NewTransformer
|
||||
initializerMap[constants.CatFileChopLumpLabel] = transformers.CatFileChopLumpTransformer.NewLogNoteTransformer
|
||||
initializerMap[constants.CatFileFlipLabel] = transformers.CatFileFlipTransformer.NewLogNoteTransformer
|
||||
initializerMap[constants.CatFilePitVowLabel] = transformers.CatFilePitVowTransformer.NewLogNoteTransformer
|
||||
initializerMap[constants.DealLabel] = transformers.DealTransformer.NewLogNoteTransformer
|
||||
initializerMap[constants.DentLabel] = transformers.DentTransformer.NewLogNoteTransformer
|
||||
initializerMap[constants.DripDripLabel] = transformers.DripDripTransformer.NewLogNoteTransformer
|
||||
initializerMap[constants.DripFileIlkLabel] = transformers.DripFileIlkTransformer.NewLogNoteTransformer
|
||||
initializerMap[constants.DripFileRepoLabel] = transformers.DripFileRepoTransformer.NewLogNoteTransformer
|
||||
initializerMap[constants.DripFileVowLabel] = transformers.DripFileVowTransfromer.NewLogNoteTransformer
|
||||
initializerMap[constants.FlapKickLabel] = transformers.FlapKickTransformer.NewTransformer
|
||||
initializerMap[constants.FlipKickLabel] = transformers.FlipKickTransformer.NewTransformer
|
||||
initializerMap[constants.FlopKickLabel] = transformers.FlopKickTransformer.NewTransformer
|
||||
initializerMap[constants.FrobLabel] = transformers.FrobTransformer.NewTransformer
|
||||
initializerMap[constants.PitFileDebtCeilingLabel] = transformers.PitFileDebtCeilingTransformer.NewLogNoteTransformer
|
||||
initializerMap[constants.PitFileIlkLabel] = transformers.PitFileIlkTransformer.NewLogNoteTransformer
|
||||
initializerMap[constants.PriceFeedLabel] = transformers.PriceFeedTransformer.NewLogNoteTransformer
|
||||
initializerMap[constants.TendLabel] = transformers.TendTransformer.NewLogNoteTransformer
|
||||
initializerMap[constants.VatFluxLabel] = transformers.VatFluxTransformer.NewLogNoteTransformer
|
||||
initializerMap[constants.VatFoldLabel] = transformers.VatFoldTransformer.NewLogNoteTransformer
|
||||
initializerMap[constants.VatGrabLabel] = transformers.VatGrabTransformer.NewLogNoteTransformer
|
||||
initializerMap[constants.VatHealLabel] = transformers.VatHealTransformer.NewLogNoteTransformer
|
||||
initializerMap[constants.VatInitLabel] = transformers.VatInitTransformer.NewLogNoteTransformer
|
||||
initializerMap[constants.VatMoveLabel] = transformers.VatMoveTransformer.NewLogNoteTransformer
|
||||
initializerMap[constants.VatSlipLabel] = transformers.VatSlipTransformer.NewLogNoteTransformer
|
||||
initializerMap[constants.VatTollLabel] = transformers.VatTollTransformer.NewLogNoteTransformer
|
||||
initializerMap[constants.VatTuneLabel] = transformers.VatTuneTransformer.NewLogNoteTransformer
|
||||
initializerMap[constants.VowFlogLabel] = transformers.FlogTransformer.NewLogNoteTransformer
|
||||
|
||||
return transformerInitializerMap
|
||||
return initializerMap
|
||||
}
|
||||
|
||||
func init() {
|
||||
|
||||
71
cmd/erc20.go
71
cmd/erc20.go
@ -1,71 +0,0 @@
|
||||
// Copyright 2018 Vulcanize
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block"
|
||||
"github.com/vulcanize/vulcanizedb/libraries/shared"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
)
|
||||
|
||||
// erc20Cmd represents the erc20 command
|
||||
var erc20Cmd = &cobra.Command{
|
||||
Use: "erc20",
|
||||
Short: "Fetches and persists token supply",
|
||||
Long: `Fetches the totalSupply for the configured token from each block and persists it in Vulcanize DB.
|
||||
vulcanizedb erc20 --config environments/public
|
||||
|
||||
Expects an ethereum node to be running and requires a .toml config file:
|
||||
|
||||
[database]
|
||||
name = "vulcanize_public"
|
||||
hostname = "localhost"
|
||||
port = 5432
|
||||
|
||||
[client]
|
||||
ipcPath = "/Users/user/Library/Ethereum/geth.ipc"
|
||||
`,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
watchERC20s()
|
||||
},
|
||||
}
|
||||
|
||||
func watchERC20s() {
|
||||
ticker := time.NewTicker(5 * time.Second)
|
||||
defer ticker.Stop()
|
||||
blockChain := getBlockChain()
|
||||
db, err := postgres.NewDB(databaseConfig, blockChain.Node())
|
||||
if err != nil {
|
||||
log.Fatal("Failed to initialize database.")
|
||||
}
|
||||
watcher := shared.Watcher{
|
||||
DB: *db,
|
||||
Blockchain: blockChain,
|
||||
}
|
||||
|
||||
watcher.AddTransformers(every_block.TransformerInitializers())
|
||||
for range ticker.C {
|
||||
watcher.Execute()
|
||||
}
|
||||
}
|
||||
|
||||
func init() {
|
||||
rootCmd.AddCommand(erc20Cmd)
|
||||
}
|
||||
@ -6,7 +6,7 @@ password = "vulcanize"
|
||||
port = 5432
|
||||
|
||||
[client]
|
||||
ipcPath = "http://147.75.199.135:8545"
|
||||
ipcPath = "http://kovan0.vulcanize.io:8545"
|
||||
|
||||
[datadog]
|
||||
name = "maker_vdb_staging"
|
||||
|
||||
File diff suppressed because one or more lines are too long
@ -1,27 +0,0 @@
|
||||
// Copyright 2018 Vulcanize
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package erc20_test_helpers
|
||||
|
||||
type TokenSupplyDBRow struct {
|
||||
ID int64
|
||||
Supply int64
|
||||
BlockID int64 `db:"block_id"`
|
||||
TokenAddress string `db:"token_address"`
|
||||
}
|
||||
|
||||
type TransferDBRow struct {
|
||||
ID int64 `db:"id"`
|
||||
VulcanizeLogID int64 `db:"vulcanize_log_id"`
|
||||
}
|
||||
@ -1,106 +0,0 @@
|
||||
// Copyright 2018 Vulcanize
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package mocks
|
||||
|
||||
import (
|
||||
"math/big"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/fakes"
|
||||
)
|
||||
|
||||
type Fetcher struct {
|
||||
ContractAddress string
|
||||
Abi string
|
||||
FetchedBlocks []int64
|
||||
BlockChain core.BlockChain
|
||||
supply big.Int
|
||||
}
|
||||
|
||||
func (f *Fetcher) SetSupply(supply string) {
|
||||
f.supply.SetString(supply, 10)
|
||||
}
|
||||
|
||||
func (f Fetcher) GetBlockChain() core.BlockChain {
|
||||
return f.BlockChain
|
||||
}
|
||||
|
||||
func (f *Fetcher) FetchSupplyOf(contractAbi string, contractAddress string, blockNumber int64) (big.Int, error) {
|
||||
f.Abi = contractAbi
|
||||
f.ContractAddress = contractAddress
|
||||
f.FetchedBlocks = append(f.FetchedBlocks, blockNumber)
|
||||
|
||||
accumulator := big.NewInt(1)
|
||||
f.supply.Add(&f.supply, accumulator)
|
||||
|
||||
return f.supply, nil
|
||||
}
|
||||
|
||||
type TotalSupplyRepository struct {
|
||||
TotalSuppliesCreated []every_block.TokenSupply
|
||||
MissingBlockNumbers []int64
|
||||
StartingBlock int64
|
||||
EndingBlock int64
|
||||
}
|
||||
|
||||
func (fr *TotalSupplyRepository) Create(supply every_block.TokenSupply) error {
|
||||
fr.TotalSuppliesCreated = append(fr.TotalSuppliesCreated, supply)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (fr *TotalSupplyRepository) MissingBlocks(startingBlock int64, highestBlock int64) ([]int64, error) {
|
||||
fr.StartingBlock = startingBlock
|
||||
fr.EndingBlock = highestBlock
|
||||
return fr.MissingBlockNumbers, nil
|
||||
}
|
||||
|
||||
func (fr *TotalSupplyRepository) SetMissingBlocks(missingBlocks []int64) {
|
||||
fr.MissingBlockNumbers = missingBlocks
|
||||
}
|
||||
|
||||
type FailureRepository struct {
|
||||
createFail bool
|
||||
missingBlocksFail bool
|
||||
missingBlocksNumbers []int64
|
||||
}
|
||||
|
||||
func (fr *FailureRepository) Create(supply every_block.TokenSupply) error {
|
||||
if fr.createFail {
|
||||
return fakes.FakeError
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (fr *FailureRepository) MissingBlocks(startingBlock int64, highestBlock int64) ([]int64, error) {
|
||||
if fr.missingBlocksFail {
|
||||
return []int64{}, fakes.FakeError
|
||||
} else {
|
||||
return fr.missingBlocksNumbers, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (fr *FailureRepository) SetCreateFail(fail bool) {
|
||||
fr.createFail = fail
|
||||
}
|
||||
|
||||
func (fr *FailureRepository) SetMissingBlocksFail(fail bool) {
|
||||
fr.missingBlocksFail = fail
|
||||
}
|
||||
|
||||
func (fr *FailureRepository) SetMissingBlocks(missingBlocks []int64) {
|
||||
fr.missingBlocksNumbers = missingBlocks
|
||||
}
|
||||
@ -1,19 +0,0 @@
|
||||
# ERC20 Transformers
|
||||
|
||||
## Description
|
||||
The Transformers in this directory are associated with contract functions and events that conform to the [ERC20 Token interface](https://theethereum.wiki/w/index.php/ERC20_Token_Standard#The_ERC20_Token_Standard_Interface).
|
||||
|
||||
See `libraries/shared/TransformerREADME.md` for further information regarding the Transformer interface.
|
||||
|
||||
## Configuration
|
||||
In addition to environment configuration mentioned in the main VulcanizeDB README, the ERC20 transformers also need to be configured with contract information for the desired token(s) to be watched. This configuration file is located at `./vulcanizedb/examples/erc20_watcher/config.go`.
|
||||
|
||||
## ERC20 Functions
|
||||
The `everyblock` directory contains transformers that fetch data from the contract itself, via one of the standard functions.
|
||||
|
||||
Currently, the `totalSupply` function transformer has been implemented. This transformer will fetch the total supply for the given contract address and persist `total_supply` records in the database.
|
||||
|
||||
## Running the transformers
|
||||
1. If running against a local node, make sure that the node has been started.
|
||||
1. In a separate terminal run the following command:
|
||||
`./vulcanizedb erc20 --config <config.toml>`
|
||||
@ -1,33 +0,0 @@
|
||||
// Copyright 2018 Vulcanize
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package erc20_watcher
|
||||
|
||||
import "github.com/vulcanize/vulcanizedb/examples/constants"
|
||||
|
||||
type ContractConfig struct {
|
||||
Address string
|
||||
Abi string
|
||||
FirstBlock int64
|
||||
LastBlock int64
|
||||
Name string
|
||||
}
|
||||
|
||||
var DaiConfig = ContractConfig{
|
||||
Address: constants.DaiContractAddress,
|
||||
Abi: constants.DaiAbiString,
|
||||
FirstBlock: int64(4752008),
|
||||
LastBlock: -1,
|
||||
Name: "Dai",
|
||||
}
|
||||
@ -1,33 +0,0 @@
|
||||
// Copyright 2018 Vulcanize
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package every_block_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"io/ioutil"
|
||||
)
|
||||
|
||||
func TestEveryBlock(t *testing.T) {
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "EveryBlock Suite")
|
||||
}
|
||||
|
||||
var _ = BeforeSuite(func() {
|
||||
log.SetOutput(ioutil.Discard)
|
||||
})
|
||||
@ -1,71 +0,0 @@
|
||||
// Copyright 2018 Vulcanize
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package every_block
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"math/big"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
)
|
||||
|
||||
type ERC20FetcherInterface interface {
|
||||
FetchSupplyOf(contractAbi string, contractAddress string, blockNumber int64) (big.Int, error)
|
||||
GetBlockChain() core.BlockChain
|
||||
}
|
||||
|
||||
func NewFetcher(blockchain core.BlockChain) Fetcher {
|
||||
return Fetcher{
|
||||
Blockchain: blockchain,
|
||||
}
|
||||
}
|
||||
|
||||
type Fetcher struct {
|
||||
Blockchain core.BlockChain
|
||||
ContractAbi string
|
||||
ContractAddress string
|
||||
}
|
||||
|
||||
type fetcherError struct {
|
||||
err string
|
||||
fetchMethod string
|
||||
}
|
||||
|
||||
func (fe *fetcherError) Error() string {
|
||||
return fmt.Sprintf("Error fetching %s: %s", fe.fetchMethod, fe.err)
|
||||
}
|
||||
|
||||
func newFetcherError(err error, fetchMethod string) *fetcherError {
|
||||
e := fetcherError{err.Error(), fetchMethod}
|
||||
log.Info(e.Error())
|
||||
return &e
|
||||
}
|
||||
|
||||
func (f Fetcher) FetchSupplyOf(contractAbi string, contractAddress string, blockNumber int64) (big.Int, error) {
|
||||
method := "totalSupply"
|
||||
var result = new(big.Int)
|
||||
err := f.Blockchain.FetchContractData(contractAbi, contractAddress, method, nil, &result, blockNumber)
|
||||
|
||||
if err != nil {
|
||||
return *result, newFetcherError(err, method)
|
||||
}
|
||||
|
||||
return *result, nil
|
||||
}
|
||||
|
||||
func (f Fetcher) GetBlockChain() core.BlockChain {
|
||||
return f.Blockchain
|
||||
}
|
||||
@ -1,82 +0,0 @@
|
||||
// Copyright 2018 Vulcanize
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package every_block_test
|
||||
|
||||
import (
|
||||
"math/big"
|
||||
|
||||
"github.com/ethereum/go-ethereum/ethclient"
|
||||
"github.com/ethereum/go-ethereum/rpc"
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/examples/constants"
|
||||
"github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/fakes"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/geth"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/geth/client"
|
||||
rpc2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/geth/node"
|
||||
)
|
||||
|
||||
var _ = Describe("ERC20 Fetcher", func() {
|
||||
blockNumber := int64(5502914)
|
||||
|
||||
Describe("FetchSupplyOf", func() {
|
||||
It("fetches data from the blockchain with the correct arguments", func() {
|
||||
fakeBlockchain := fakes.NewMockBlockChain()
|
||||
testFetcher := every_block.NewFetcher(fakeBlockchain)
|
||||
testAbi := "testAbi"
|
||||
testContractAddress := "testContractAddress"
|
||||
_, err := testFetcher.FetchSupplyOf(testAbi, testContractAddress, blockNumber)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
expectedResult := big.Int{}
|
||||
expected := &expectedResult
|
||||
fakeBlockchain.AssertFetchContractDataCalledWith(testAbi, testContractAddress, "totalSupply", nil, &expected, blockNumber)
|
||||
})
|
||||
|
||||
It("fetches a token's total supply at the given block height", func() {
|
||||
infuraIPC := "https://mainnet.infura.io/J5Vd2fRtGsw0zZ0Ov3BL"
|
||||
rawRpcClient, err := rpc.Dial(infuraIPC)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
rpcClient := client.NewRpcClient(rawRpcClient, infuraIPC)
|
||||
ethClient := ethclient.NewClient(rawRpcClient)
|
||||
blockChainClient := client.NewEthClient(ethClient)
|
||||
node := node.MakeNode(rpcClient)
|
||||
transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
|
||||
blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
|
||||
realFetcher := every_block.NewFetcher(blockChain)
|
||||
result, err := realFetcher.FetchSupplyOf(constants.DaiAbiString, constants.DaiContractAddress, blockNumber)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
expectedResult := big.Int{}
|
||||
expectedResult.SetString("27647235749155415536952630", 10)
|
||||
Expect(result).To(Equal(expectedResult))
|
||||
})
|
||||
|
||||
It("returns an error if the call to the blockchain fails", func() {
|
||||
blockChain := fakes.NewMockBlockChain()
|
||||
blockChain.SetFetchContractDataErr(fakes.FakeError)
|
||||
errorFetcher := every_block.NewFetcher(blockChain)
|
||||
result, err := errorFetcher.FetchSupplyOf("", "", 0)
|
||||
|
||||
Expect(result.String()).To(Equal("0"))
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("totalSupply"))
|
||||
Expect(err.Error()).To(ContainSubstring(fakes.FakeError.Error()))
|
||||
})
|
||||
})
|
||||
})
|
||||
@ -1,83 +0,0 @@
|
||||
// Copyright 2018 Vulcanize
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package every_block_test
|
||||
|
||||
import (
|
||||
"math/big"
|
||||
"strconv"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/examples/constants"
|
||||
"github.com/vulcanize/vulcanizedb/examples/erc20_test_helpers"
|
||||
"github.com/vulcanize/vulcanizedb/examples/erc20_watcher"
|
||||
"github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/fakes"
|
||||
"github.com/vulcanize/vulcanizedb/test_config"
|
||||
)
|
||||
|
||||
func setLastBlockOnChain(blockChain *fakes.MockBlockChain, blockNumber int64) {
|
||||
blockNumberString := strconv.FormatInt(blockNumber, 10)
|
||||
lastBlockOnChain := big.Int{}
|
||||
lastBlockOnChain.SetString(blockNumberString, 10)
|
||||
blockChain.SetLastBlock(&lastBlockOnChain)
|
||||
}
|
||||
|
||||
var _ = Describe("Everyblock transformers", func() {
|
||||
var db *postgres.DB
|
||||
var blockChain *fakes.MockBlockChain
|
||||
var blockNumber int64
|
||||
var blockId int64
|
||||
var err error
|
||||
|
||||
BeforeEach(func() {
|
||||
blockChain = fakes.NewMockBlockChain()
|
||||
blockNumber = erc20_watcher.DaiConfig.FirstBlock
|
||||
lastBlockNumber := blockNumber + 1
|
||||
node := test_config.NewTestNode()
|
||||
db = test_config.NewTestDB(node)
|
||||
test_config.CleanTestDB(db)
|
||||
setLastBlockOnChain(blockChain, lastBlockNumber)
|
||||
|
||||
blockRepository := repositories.NewBlockRepository(db)
|
||||
|
||||
blockId, err = blockRepository.CreateOrUpdateBlock(core.Block{Number: blockNumber})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
_, err = blockRepository.CreateOrUpdateBlock(core.Block{Number: lastBlockNumber})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
})
|
||||
|
||||
It("creates a token_supply record for each block in the given range", func() {
|
||||
initializer := every_block.TokenSupplyTransformerInitializer{Config: erc20_watcher.DaiConfig}
|
||||
transformer := initializer.NewTokenSupplyTransformer(db, blockChain)
|
||||
transformer.Execute()
|
||||
|
||||
var tokenSupplyCount int
|
||||
err := db.QueryRow(`SELECT COUNT(*) FROM token_supply where block_id = $1`, blockId).Scan(&tokenSupplyCount)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(tokenSupplyCount).To(Equal(1))
|
||||
|
||||
var tokenSupply erc20_test_helpers.TokenSupplyDBRow
|
||||
err = db.Get(&tokenSupply, `SELECT * from token_supply where block_id = $1`, blockId)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(tokenSupply.BlockID).To(Equal(blockId))
|
||||
Expect(tokenSupply.TokenAddress).To(Equal(constants.DaiContractAddress))
|
||||
Expect(tokenSupply.Supply).To(Equal(int64(0)))
|
||||
})
|
||||
})
|
||||
@ -1,21 +0,0 @@
|
||||
// Copyright 2018 Vulcanize
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package every_block
|
||||
|
||||
type TokenSupply struct {
|
||||
Value string
|
||||
TokenAddress string
|
||||
BlockNumber int64
|
||||
}
|
||||
@ -1,91 +0,0 @@
|
||||
// Copyright 2018 Vulcanize
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package every_block
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
)
|
||||
|
||||
type ERC20RepositoryInterface interface {
|
||||
Create(supply TokenSupply) error
|
||||
MissingBlocks(startingBlock int64, highestBlock int64) ([]int64, error)
|
||||
}
|
||||
|
||||
type TokenSupplyRepository struct {
|
||||
*postgres.DB
|
||||
}
|
||||
|
||||
type repositoryError struct {
|
||||
err string
|
||||
msg string
|
||||
blockNumber int64
|
||||
}
|
||||
|
||||
func (re *repositoryError) Error() string {
|
||||
return fmt.Sprintf(re.msg, re.blockNumber, re.err)
|
||||
}
|
||||
|
||||
func newRepositoryError(err error, msg string, blockNumber int64) error {
|
||||
e := repositoryError{err.Error(), msg, blockNumber}
|
||||
log.Println(e.Error())
|
||||
return &e
|
||||
}
|
||||
|
||||
const (
|
||||
GetBlockError = "Error fetching block number %d: %s"
|
||||
InsertTokenSupplyError = "Error inserting token_supply for block number %d: %s"
|
||||
MissingBlockError = "Error finding missing token_supply records starting at block %d: %s"
|
||||
)
|
||||
|
||||
func (tsp *TokenSupplyRepository) Create(supply TokenSupply) error {
|
||||
var blockId int
|
||||
err := tsp.DB.Get(&blockId, `SELECT id FROM blocks WHERE number = $1 AND eth_node_id = $2`, supply.BlockNumber, tsp.NodeID)
|
||||
if err != nil {
|
||||
return newRepositoryError(err, GetBlockError, supply.BlockNumber)
|
||||
}
|
||||
|
||||
_, err = tsp.DB.Exec(
|
||||
`INSERT INTO token_supply (supply, token_address, block_id)
|
||||
VALUES($1, $2, $3)`,
|
||||
supply.Value, supply.TokenAddress, blockId)
|
||||
if err != nil {
|
||||
return newRepositoryError(err, InsertTokenSupplyError, supply.BlockNumber)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tsp *TokenSupplyRepository) MissingBlocks(startingBlock int64, highestBlock int64) ([]int64, error) {
|
||||
blockNumbers := make([]int64, 0)
|
||||
|
||||
err := tsp.DB.Select(
|
||||
&blockNumbers,
|
||||
`SELECT number FROM BLOCKS
|
||||
LEFT JOIN token_supply ON blocks.id = block_id
|
||||
WHERE block_id ISNULL
|
||||
AND eth_node_id = $1
|
||||
AND number >= $2
|
||||
AND number <= $3
|
||||
LIMIT 20`,
|
||||
tsp.NodeID,
|
||||
startingBlock,
|
||||
highestBlock,
|
||||
)
|
||||
if err != nil {
|
||||
return []int64{}, newRepositoryError(err, MissingBlockError, startingBlock)
|
||||
}
|
||||
return blockNumbers, err
|
||||
}
|
||||
@ -1,207 +0,0 @@
|
||||
// Copyright 2018 Vulcanize
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package every_block_test
|
||||
|
||||
import (
|
||||
"math/rand"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/examples/erc20_test_helpers"
|
||||
"github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
|
||||
"github.com/vulcanize/vulcanizedb/test_config"
|
||||
)
|
||||
|
||||
var _ = Describe("ERC20 Token Repository", func() {
|
||||
var db *postgres.DB
|
||||
var blockId int64
|
||||
var blockNumber int64
|
||||
var repository every_block.TokenSupplyRepository
|
||||
var blockRepository repositories.BlockRepository
|
||||
testAddress := "abc"
|
||||
|
||||
BeforeEach(func() {
|
||||
node := test_config.NewTestNode()
|
||||
db = test_config.NewTestDB(node)
|
||||
test_config.CleanTestDB(db)
|
||||
repository = every_block.TokenSupplyRepository{DB: db}
|
||||
blockRepository = *repositories.NewBlockRepository(db)
|
||||
blockNumber = rand.Int63()
|
||||
blockId = test_config.NewTestBlock(blockNumber, blockRepository)
|
||||
})
|
||||
|
||||
Describe("Create", func() {
|
||||
It("creates a token supply record", func() {
|
||||
supply := supplyModel(blockNumber, testAddress, "100")
|
||||
err := repository.Create(supply)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
dbResult := erc20_test_helpers.TokenSupplyDBRow{}
|
||||
expectedTokenSupply := erc20_test_helpers.TokenSupplyDBRow{
|
||||
Supply: int64(100),
|
||||
BlockID: blockId,
|
||||
TokenAddress: testAddress,
|
||||
}
|
||||
|
||||
var count int
|
||||
err = repository.DB.QueryRowx(`SELECT count(*) FROM token_supply`).Scan(&count)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(count).To(Equal(1))
|
||||
|
||||
err = repository.DB.QueryRowx(`SELECT * FROM token_supply`).StructScan(&dbResult)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(dbResult.Supply).To(Equal(expectedTokenSupply.Supply))
|
||||
Expect(dbResult.BlockID).To(Equal(expectedTokenSupply.BlockID))
|
||||
Expect(dbResult.TokenAddress).To(Equal(expectedTokenSupply.TokenAddress))
|
||||
})
|
||||
|
||||
It("returns an error if fetching the block's id from the database fails", func() {
|
||||
errorSupply := supplyModel(-1, "", "")
|
||||
err := repository.Create(errorSupply)
|
||||
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("sql"))
|
||||
Expect(err.Error()).To(ContainSubstring("block number -1"))
|
||||
})
|
||||
|
||||
It("returns an error if inserting the token_supply fails", func() {
|
||||
errorSupply := supplyModel(blockNumber, "", "")
|
||||
err := repository.Create(errorSupply)
|
||||
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("pq"))
|
||||
Expect(err.Error()).To(ContainSubstring("token_supply for block number"))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("When there are multiple nodes", func() {
|
||||
var node2DB *postgres.DB
|
||||
var node2BlockRepo *repositories.BlockRepository
|
||||
var node2BlockId int64
|
||||
var node2TokenSupplyRepo every_block.TokenSupplyRepository
|
||||
var tokenSupply every_block.TokenSupply
|
||||
|
||||
BeforeEach(func() {
|
||||
node2DB = createDbForAnotherNode()
|
||||
|
||||
//create another block with the same number on node2
|
||||
node2BlockRepo = repositories.NewBlockRepository(node2DB)
|
||||
node2BlockId = test_config.NewTestBlock(blockNumber, *node2BlockRepo)
|
||||
|
||||
tokenSupply = supplyModel(blockNumber, "abc", "100")
|
||||
node2TokenSupplyRepo = every_block.TokenSupplyRepository{DB: node2DB}
|
||||
})
|
||||
|
||||
It("only creates token_supply records for the current node (node2)", func() {
|
||||
err := node2TokenSupplyRepo.Create(tokenSupply)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
var tokenSupplies []erc20_test_helpers.TokenSupplyDBRow
|
||||
err = node2TokenSupplyRepo.DB.Select(&tokenSupplies, `SELECT * FROM token_supply`)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(len(tokenSupplies)).To(Equal(1))
|
||||
Expect(tokenSupplies[0].BlockID).To(Equal(node2BlockId))
|
||||
})
|
||||
|
||||
It("only includes missing block numbers for the current node", func() {
|
||||
//create token_supply on original node
|
||||
err := repository.Create(tokenSupply)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
originalNodeMissingBlocks, err := repository.MissingBlocks(blockNumber, blockNumber)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(len(originalNodeMissingBlocks)).To(Equal(0))
|
||||
|
||||
node2MissingBlocks, err := node2TokenSupplyRepo.MissingBlocks(blockNumber, blockNumber)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(len(node2MissingBlocks)).To(Equal(1))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("MissingBlocks", func() {
|
||||
It("returns the block numbers for which an associated TokenSupply record hasn't been created", func() {
|
||||
createTokenSupplyFor(repository, blockNumber)
|
||||
|
||||
newBlockNumber := blockNumber + 1
|
||||
test_config.NewTestBlock(newBlockNumber, blockRepository)
|
||||
blocks, err := repository.MissingBlocks(blockNumber, newBlockNumber)
|
||||
|
||||
Expect(blocks).To(ConsistOf(newBlockNumber))
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
})
|
||||
|
||||
It("only returns blocks within the given range", func() {
|
||||
newBlockNumber := blockNumber + 1
|
||||
test_config.NewTestBlock(newBlockNumber, blockRepository)
|
||||
blocks, err := repository.MissingBlocks(blockNumber, blockNumber)
|
||||
|
||||
Expect(blocks).NotTo(ConsistOf(newBlockNumber))
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
})
|
||||
|
||||
It("does not return numbers that already have an associated TokenSupply record", func() {
|
||||
createTokenSupplyFor(repository, blockNumber)
|
||||
blocks, err := repository.MissingBlocks(blockNumber, blockNumber)
|
||||
|
||||
Expect(blocks).To(BeEmpty())
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
})
|
||||
})
|
||||
|
||||
It("deletes the token supply record when the associated block is deleted", func() {
|
||||
err := repository.Create(every_block.TokenSupply{BlockNumber: blockNumber, Value: "0"})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
var count int
|
||||
err = repository.DB.QueryRowx(`SELECT count(*) FROM token_supply`).Scan(&count)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(count).To(Equal(1))
|
||||
|
||||
_, err = db.Query(`DELETE FROM blocks`)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
err = repository.DB.QueryRowx(`SELECT count(*) FROM token_supply`).Scan(&count)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(count).To(Equal(0))
|
||||
})
|
||||
})
|
||||
|
||||
func supplyModel(blockNumber int64, tokenAddress string, supplyValue string) every_block.TokenSupply {
|
||||
return every_block.TokenSupply{
|
||||
Value: supplyValue,
|
||||
TokenAddress: tokenAddress,
|
||||
BlockNumber: int64(blockNumber),
|
||||
}
|
||||
}
|
||||
|
||||
func createTokenSupplyFor(repository every_block.TokenSupplyRepository, blockNumber int64) {
|
||||
err := repository.Create(every_block.TokenSupply{BlockNumber: blockNumber, Value: "0"})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
}
|
||||
|
||||
func createDbForAnotherNode() *postgres.DB {
|
||||
anotherNode := core.Node{
|
||||
GenesisBlock: "GENESIS",
|
||||
NetworkID: 1,
|
||||
ID: "testNodeId",
|
||||
ClientName: "Geth/v1.7.2-stable-1db4ecdc/darwin-amd64/go1.9",
|
||||
}
|
||||
|
||||
return test_config.NewTestDB(anotherNode)
|
||||
}
|
||||
@ -1,118 +0,0 @@
|
||||
// Copyright 2018 Vulcanize
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package every_block
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/big"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/examples/erc20_watcher"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
|
||||
)
|
||||
|
||||
type Transformer struct {
|
||||
Fetcher ERC20FetcherInterface
|
||||
Repository ERC20RepositoryInterface
|
||||
Config erc20_watcher.ContractConfig
|
||||
}
|
||||
|
||||
func (t *Transformer) SetConfiguration(config erc20_watcher.ContractConfig) {
|
||||
t.Config = config
|
||||
}
|
||||
|
||||
type TokenSupplyTransformerInitializer struct {
|
||||
Config erc20_watcher.ContractConfig
|
||||
}
|
||||
|
||||
func (i TokenSupplyTransformerInitializer) NewTokenSupplyTransformer(db *postgres.DB, blockchain core.BlockChain) shared.Transformer {
|
||||
fetcher := NewFetcher(blockchain)
|
||||
repository := TokenSupplyRepository{DB: db}
|
||||
transformer := Transformer{
|
||||
Fetcher: &fetcher,
|
||||
Repository: &repository,
|
||||
Config: i.Config,
|
||||
}
|
||||
|
||||
return transformer
|
||||
}
|
||||
|
||||
const (
|
||||
FetchingBlocksError = "Error fetching missing blocks starting at block number %d: %s"
|
||||
FetchingSupplyError = "Error fetching supply for block %d: %s"
|
||||
CreateSupplyError = "Error inserting token_supply for block %d: %s"
|
||||
)
|
||||
|
||||
type transformerError struct {
|
||||
err string
|
||||
blockNumber int64
|
||||
msg string
|
||||
}
|
||||
|
||||
func (te *transformerError) Error() string {
|
||||
return fmt.Sprintf(te.msg, te.blockNumber, te.err)
|
||||
}
|
||||
|
||||
func newTransformerError(err error, blockNumber int64, msg string) error {
|
||||
e := transformerError{err.Error(), blockNumber, msg}
|
||||
log.Println(e.Error())
|
||||
return &e
|
||||
}
|
||||
|
||||
func (t Transformer) Execute() error {
|
||||
var upperBoundBlock int64
|
||||
blockchain := t.Fetcher.GetBlockChain()
|
||||
lastBlock := blockchain.LastBlock().Int64()
|
||||
|
||||
if t.Config.LastBlock == -1 {
|
||||
upperBoundBlock = lastBlock
|
||||
} else {
|
||||
upperBoundBlock = t.Config.LastBlock
|
||||
}
|
||||
|
||||
blocks, err := t.Repository.MissingBlocks(t.Config.FirstBlock, upperBoundBlock)
|
||||
|
||||
if err != nil {
|
||||
return newTransformerError(err, t.Config.FirstBlock, FetchingBlocksError)
|
||||
}
|
||||
|
||||
log.Printf("Fetching totalSupply for %d blocks", len(blocks))
|
||||
for _, blockNumber := range blocks {
|
||||
totalSupply, err := t.Fetcher.FetchSupplyOf(t.Config.Abi, t.Config.Address, blockNumber)
|
||||
|
||||
if err != nil {
|
||||
return newTransformerError(err, blockNumber, FetchingSupplyError)
|
||||
}
|
||||
model := createTokenSupplyModel(totalSupply, t.Config.Address, blockNumber)
|
||||
err = t.Repository.Create(model)
|
||||
|
||||
if err != nil {
|
||||
return newTransformerError(err, blockNumber, CreateSupplyError)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func createTokenSupplyModel(totalSupply big.Int, address string, blockNumber int64) TokenSupply {
|
||||
return TokenSupply{
|
||||
Value: totalSupply.String(),
|
||||
TokenAddress: address,
|
||||
BlockNumber: blockNumber,
|
||||
}
|
||||
}
|
||||
@ -1,188 +0,0 @@
|
||||
// Copyright 2018 Vulcanize
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package every_block_test
|
||||
|
||||
import (
|
||||
"math/big"
|
||||
"math/rand"
|
||||
"strconv"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/examples/constants"
|
||||
"github.com/vulcanize/vulcanizedb/examples/erc20_test_helpers/mocks"
|
||||
"github.com/vulcanize/vulcanizedb/examples/erc20_watcher"
|
||||
"github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/fakes"
|
||||
)
|
||||
|
||||
var testContractConfig = erc20_watcher.ContractConfig{
|
||||
Address: constants.DaiContractAddress,
|
||||
Abi: constants.DaiAbiString,
|
||||
FirstBlock: int64(4752008),
|
||||
LastBlock: int64(5750050),
|
||||
Name: "Dai",
|
||||
}
|
||||
|
||||
var config = testContractConfig
|
||||
|
||||
var _ = Describe("Everyblock transformer", func() {
|
||||
var fetcher mocks.Fetcher
|
||||
var repository mocks.TotalSupplyRepository
|
||||
var transformer every_block.Transformer
|
||||
var blockChain *fakes.MockBlockChain
|
||||
var initialSupply = "27647235749155415536952630"
|
||||
var initialSupplyPlusOne = "27647235749155415536952631"
|
||||
var initialSupplyPlusTwo = "27647235749155415536952632"
|
||||
var initialSupplyPlusThree = "27647235749155415536952633"
|
||||
var defaultLastBlock = big.Int{}
|
||||
|
||||
BeforeEach(func() {
|
||||
blockChain = fakes.NewMockBlockChain()
|
||||
blockChain.SetLastBlock(&defaultLastBlock)
|
||||
fetcher = mocks.Fetcher{BlockChain: blockChain}
|
||||
fetcher.SetSupply(initialSupply)
|
||||
repository = mocks.TotalSupplyRepository{}
|
||||
repository.SetMissingBlocks([]int64{config.FirstBlock})
|
||||
//setting the mock repository to return the first block as the missing blocks
|
||||
|
||||
transformer = every_block.Transformer{
|
||||
Fetcher: &fetcher,
|
||||
Repository: &repository,
|
||||
}
|
||||
transformer.SetConfiguration(config)
|
||||
})
|
||||
|
||||
It("fetches and persists the total supply of a token for a single block", func() {
|
||||
err := transformer.Execute()
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
Expect(len(fetcher.FetchedBlocks)).To(Equal(1))
|
||||
Expect(fetcher.FetchedBlocks).To(ConsistOf(config.FirstBlock))
|
||||
Expect(fetcher.Abi).To(Equal(config.Abi))
|
||||
Expect(fetcher.ContractAddress).To(Equal(config.Address))
|
||||
|
||||
Expect(repository.StartingBlock).To(Equal(config.FirstBlock))
|
||||
Expect(repository.EndingBlock).To(Equal(config.LastBlock))
|
||||
Expect(len(repository.TotalSuppliesCreated)).To(Equal(1))
|
||||
expectedSupply := big.Int{}
|
||||
expectedSupply.SetString(initialSupply, 10)
|
||||
expectedSupply.Add(&expectedSupply, big.NewInt(1))
|
||||
|
||||
Expect(repository.TotalSuppliesCreated[0].Value).To(Equal(expectedSupply.String()))
|
||||
})
|
||||
|
||||
It("retrieves the total supply for every missing block", func() {
|
||||
missingBlocks := []int64{
|
||||
config.FirstBlock,
|
||||
config.FirstBlock + 1,
|
||||
config.FirstBlock + 2,
|
||||
}
|
||||
repository.SetMissingBlocks(missingBlocks)
|
||||
transformer.Execute()
|
||||
|
||||
Expect(len(fetcher.FetchedBlocks)).To(Equal(3))
|
||||
Expect(fetcher.FetchedBlocks).To(ConsistOf(config.FirstBlock, config.FirstBlock+1, config.FirstBlock+2))
|
||||
Expect(fetcher.Abi).To(Equal(config.Abi))
|
||||
Expect(fetcher.ContractAddress).To(Equal(config.Address))
|
||||
|
||||
Expect(len(repository.TotalSuppliesCreated)).To(Equal(3))
|
||||
Expect(repository.TotalSuppliesCreated[0].Value).To(Equal(initialSupplyPlusOne))
|
||||
Expect(repository.TotalSuppliesCreated[1].Value).To(Equal(initialSupplyPlusTwo))
|
||||
Expect(repository.TotalSuppliesCreated[2].Value).To(Equal(initialSupplyPlusThree))
|
||||
})
|
||||
|
||||
It("uses the set contract configuration", func() {
|
||||
repository.SetMissingBlocks([]int64{testContractConfig.FirstBlock})
|
||||
transformer.SetConfiguration(testContractConfig)
|
||||
err := transformer.Execute()
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
Expect(fetcher.FetchedBlocks).To(ConsistOf(testContractConfig.FirstBlock))
|
||||
Expect(fetcher.Abi).To(Equal(testContractConfig.Abi))
|
||||
Expect(fetcher.ContractAddress).To(Equal(testContractConfig.Address))
|
||||
|
||||
Expect(repository.StartingBlock).To(Equal(testContractConfig.FirstBlock))
|
||||
Expect(repository.EndingBlock).To(Equal(testContractConfig.LastBlock))
|
||||
Expect(len(repository.TotalSuppliesCreated)).To(Equal(1))
|
||||
expectedTokenSupply := every_block.TokenSupply{
|
||||
Value: initialSupplyPlusOne,
|
||||
TokenAddress: testContractConfig.Address,
|
||||
BlockNumber: testContractConfig.FirstBlock,
|
||||
}
|
||||
Expect(repository.TotalSuppliesCreated[0]).To(Equal(expectedTokenSupply))
|
||||
})
|
||||
|
||||
It("uses the most recent block if the Config.LastBlock is -1", func() {
|
||||
testContractConfig.LastBlock = -1
|
||||
transformer.SetConfiguration(testContractConfig)
|
||||
|
||||
randomBlockNumber := rand.Int63()
|
||||
numberToString := strconv.FormatInt(randomBlockNumber, 10)
|
||||
mostRecentBlock := big.Int{}
|
||||
mostRecentBlock.SetString(numberToString, 10)
|
||||
|
||||
blockChain.SetLastBlock(&mostRecentBlock)
|
||||
|
||||
err := transformer.Execute()
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
Expect(repository.EndingBlock).To(Equal(randomBlockNumber))
|
||||
})
|
||||
|
||||
It("returns an error if the call to get missing blocks fails", func() {
|
||||
failureRepository := mocks.FailureRepository{}
|
||||
failureRepository.SetMissingBlocksFail(true)
|
||||
transformer = every_block.Transformer{
|
||||
Fetcher: &fetcher,
|
||||
Repository: &failureRepository,
|
||||
}
|
||||
err := transformer.Execute()
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring(fakes.FakeError.Error()))
|
||||
Expect(err.Error()).To(ContainSubstring("fetching missing blocks"))
|
||||
})
|
||||
|
||||
It("returns an error if the call to the blockChain fails", func() {
|
||||
failureBlockchain := fakes.NewMockBlockChain()
|
||||
failureBlockchain.SetLastBlock(&defaultLastBlock)
|
||||
failureBlockchain.SetFetchContractDataErr(fakes.FakeError)
|
||||
fetcher := every_block.NewFetcher(failureBlockchain)
|
||||
transformer = every_block.Transformer{
|
||||
Fetcher: &fetcher,
|
||||
Repository: &repository,
|
||||
}
|
||||
err := transformer.Execute()
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring(fakes.FakeError.Error()))
|
||||
Expect(err.Error()).To(ContainSubstring("supply"))
|
||||
})
|
||||
|
||||
It("returns an error if the call to save the token_supply fails", func() {
|
||||
failureRepository := mocks.FailureRepository{}
|
||||
failureRepository.SetMissingBlocks([]int64{config.FirstBlock})
|
||||
failureRepository.SetCreateFail(true)
|
||||
|
||||
transformer = every_block.Transformer{
|
||||
Fetcher: &fetcher,
|
||||
Repository: &failureRepository,
|
||||
}
|
||||
err := transformer.Execute()
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring(fakes.FakeError.Error()))
|
||||
Expect(err.Error()).To(ContainSubstring("supply"))
|
||||
})
|
||||
})
|
||||
@ -1,28 +0,0 @@
|
||||
// Copyright 2018 Vulcanize
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package every_block
|
||||
|
||||
import (
|
||||
"github.com/vulcanize/vulcanizedb/examples/erc20_watcher"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
|
||||
)
|
||||
|
||||
func TransformerInitializers() []shared.TransformerInitializer {
|
||||
config := erc20_watcher.DaiConfig
|
||||
initializer := TokenSupplyTransformerInitializer{config}
|
||||
return []shared.TransformerInitializer{
|
||||
initializer.NewTokenSupplyTransformer,
|
||||
}
|
||||
}
|
||||
@ -1,6 +1,8 @@
|
||||
package shared
|
||||
|
||||
import (
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/log"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
|
||||
@ -8,21 +10,82 @@ import (
|
||||
|
||||
type Watcher struct {
|
||||
Transformers []shared.Transformer
|
||||
DB postgres.DB
|
||||
Blockchain core.BlockChain
|
||||
DB *postgres.DB
|
||||
Fetcher shared.LogFetcher
|
||||
Chunker shared.Chunker
|
||||
Addresses []common.Address
|
||||
Topics []common.Hash
|
||||
}
|
||||
|
||||
func (watcher *Watcher) AddTransformers(us []shared.TransformerInitializer) {
|
||||
for _, transformerInitializer := range us {
|
||||
transformer := transformerInitializer(&watcher.DB, watcher.Blockchain)
|
||||
watcher.Transformers = append(watcher.Transformers, transformer)
|
||||
func NewWatcher(db *postgres.DB, bc core.BlockChain) Watcher {
|
||||
chunker := shared.NewLogChunker()
|
||||
fetcher := shared.NewFetcher(bc)
|
||||
return Watcher{
|
||||
DB: db,
|
||||
Fetcher: fetcher,
|
||||
Chunker: chunker,
|
||||
}
|
||||
}
|
||||
|
||||
// Adds transformers to the watcher and updates the chunker, so that it will consider the new transformers.
|
||||
func (watcher *Watcher) AddTransformers(initializers []shared.TransformerInitializer) {
|
||||
var contractAddresses []common.Address
|
||||
var topic0s []common.Hash
|
||||
var configs []shared.TransformerConfig
|
||||
|
||||
for _, initializer := range initializers {
|
||||
transformer := initializer(watcher.DB)
|
||||
watcher.Transformers = append(watcher.Transformers, transformer)
|
||||
|
||||
config := transformer.GetConfig()
|
||||
configs = append(configs, config)
|
||||
|
||||
addresses := shared.HexStringsToAddresses(config.ContractAddresses)
|
||||
contractAddresses = append(contractAddresses, addresses...)
|
||||
topic0s = append(topic0s, common.HexToHash(config.Topic))
|
||||
}
|
||||
|
||||
watcher.Addresses = append(watcher.Addresses, contractAddresses...)
|
||||
watcher.Topics = append(watcher.Topics, topic0s...)
|
||||
watcher.Chunker.AddConfigs(configs)
|
||||
}
|
||||
|
||||
func (watcher *Watcher) Execute() error {
|
||||
var err error
|
||||
for _, transformer := range watcher.Transformers {
|
||||
err = transformer.Execute()
|
||||
checkedColumnNames, err := shared.GetCheckedColumnNames(watcher.DB)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
notCheckedSQL := shared.CreateNotCheckedSQL(checkedColumnNames)
|
||||
|
||||
// TODO Handle start and end numbers in transformers
|
||||
missingHeaders, err := shared.MissingHeaders(0, -1, watcher.DB, notCheckedSQL)
|
||||
if err != nil {
|
||||
log.Error("Fetching of missing headers failed in watcher!")
|
||||
return err
|
||||
}
|
||||
|
||||
for _, header := range missingHeaders {
|
||||
// TODO Extend FetchLogs for doing several blocks at a time
|
||||
logs, err := watcher.Fetcher.FetchLogs(watcher.Addresses, watcher.Topics, header)
|
||||
if err != nil {
|
||||
// TODO Handle fetch error in watcher
|
||||
log.Error("Error while fetching logs for header %v in watcher", header.Id)
|
||||
return err
|
||||
}
|
||||
|
||||
chunkedLogs := watcher.Chunker.ChunkLogs(logs)
|
||||
|
||||
// Can't quit early and mark as checked if there are no logs. If we are running continuousLogSync,
|
||||
// not all logs we're interested in might have been fetched.
|
||||
for _, transformer := range watcher.Transformers {
|
||||
transformerName := transformer.GetConfig().TransformerName
|
||||
logChunk := chunkedLogs[transformerName]
|
||||
err = transformer.Execute(logChunk, header)
|
||||
if err != nil {
|
||||
log.Error("%v transformer failed to execute in watcher: %v", transformerName, err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
@ -2,69 +2,162 @@ package shared_test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"github.com/ethereum/go-ethereum"
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/vulcanize/vulcanizedb/libraries/shared"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/fakes"
|
||||
shared2 "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks"
|
||||
"github.com/vulcanize/vulcanizedb/test_config"
|
||||
)
|
||||
|
||||
type MockTransformer struct {
|
||||
executeWasCalled bool
|
||||
executeError error
|
||||
}
|
||||
|
||||
func (mh *MockTransformer) Execute() error {
|
||||
if mh.executeError != nil {
|
||||
return mh.executeError
|
||||
}
|
||||
mh.executeWasCalled = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func fakeTransformerInitializer(db *postgres.DB, blockchain core.BlockChain) shared2.Transformer {
|
||||
return &MockTransformer{}
|
||||
}
|
||||
|
||||
var _ = Describe("Watcher", func() {
|
||||
It("Adds transformers", func() {
|
||||
watcher := shared.Watcher{}
|
||||
It("initialises correctly", func() {
|
||||
db := test_config.NewTestDB(core.Node{ID: "testNode"})
|
||||
bc := fakes.NewMockBlockChain()
|
||||
|
||||
watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformerInitializer})
|
||||
watcher := shared.NewWatcher(db, bc)
|
||||
|
||||
Expect(watcher.DB).To(Equal(db))
|
||||
Expect(watcher.Fetcher).NotTo(BeNil())
|
||||
Expect(watcher.Chunker).NotTo(BeNil())
|
||||
})
|
||||
|
||||
It("adds transformers", func() {
|
||||
watcher := shared.NewWatcher(nil, nil)
|
||||
fakeTransformer := &mocks.MockTransformer{}
|
||||
fakeTransformer.SetTransformerConfig(mocks.FakeTransformerConfig)
|
||||
watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformer.FakeTransformerInitializer})
|
||||
|
||||
Expect(len(watcher.Transformers)).To(Equal(1))
|
||||
Expect(watcher.Transformers).To(ConsistOf(&MockTransformer{}))
|
||||
Expect(watcher.Transformers).To(ConsistOf(fakeTransformer))
|
||||
Expect(watcher.Topics).To(Equal([]common.Hash{common.HexToHash("FakeTopic")}))
|
||||
Expect(watcher.Addresses).To(Equal([]common.Address{common.HexToAddress("FakeAddress")}))
|
||||
})
|
||||
|
||||
It("Adds transformers from multiple sources", func() {
|
||||
watcher := shared.Watcher{}
|
||||
It("adds transformers from multiple sources", func() {
|
||||
watcher := shared.NewWatcher(nil, nil)
|
||||
fakeTransformer1 := &mocks.MockTransformer{}
|
||||
fakeTransformer1.SetTransformerConfig(mocks.FakeTransformerConfig)
|
||||
|
||||
watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformerInitializer})
|
||||
watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformerInitializer})
|
||||
fakeTransformer2 := &mocks.MockTransformer{}
|
||||
fakeTransformer2.SetTransformerConfig(mocks.FakeTransformerConfig)
|
||||
|
||||
watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformer1.FakeTransformerInitializer})
|
||||
watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformer2.FakeTransformerInitializer})
|
||||
|
||||
Expect(len(watcher.Transformers)).To(Equal(2))
|
||||
Expect(watcher.Topics).To(Equal([]common.Hash{common.HexToHash("FakeTopic"),
|
||||
common.HexToHash("FakeTopic")}))
|
||||
Expect(watcher.Addresses).To(Equal([]common.Address{common.HexToAddress("FakeAddress"),
|
||||
common.HexToAddress("FakeAddress")}))
|
||||
})
|
||||
|
||||
It("Executes each transformer", func() {
|
||||
watcher := shared.Watcher{}
|
||||
fakeTransformer := &MockTransformer{}
|
||||
watcher.Transformers = []shared2.Transformer{fakeTransformer}
|
||||
Describe("with missing headers", func() {
|
||||
var (
|
||||
db *postgres.DB
|
||||
watcher shared.Watcher
|
||||
mockBlockChain fakes.MockBlockChain
|
||||
fakeTransformer *mocks.MockTransformer
|
||||
headerRepository repositories.HeaderRepository
|
||||
repository mocks.MockWatcherRepository
|
||||
)
|
||||
|
||||
watcher.Execute()
|
||||
BeforeEach(func() {
|
||||
db = test_config.NewTestDB(test_config.NewTestNode())
|
||||
test_config.CleanTestDB(db)
|
||||
mockBlockChain = fakes.MockBlockChain{}
|
||||
headerRepository = repositories.NewHeaderRepository(db)
|
||||
_, err := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
Expect(fakeTransformer.executeWasCalled).To(BeTrue())
|
||||
})
|
||||
repository = mocks.MockWatcherRepository{}
|
||||
watcher = shared.NewWatcher(db, &mockBlockChain)
|
||||
})
|
||||
|
||||
It("Returns an error if transformer returns an error", func() {
|
||||
watcher := shared.Watcher{}
|
||||
fakeTransformer := &MockTransformer{executeError: errors.New("Something bad happened")}
|
||||
watcher.Transformers = []shared2.Transformer{fakeTransformer}
|
||||
It("executes each transformer", func() {
|
||||
fakeTransformer = &mocks.MockTransformer{}
|
||||
watcher.Transformers = []shared2.Transformer{fakeTransformer}
|
||||
repository.SetMissingHeaders([]core.Header{fakes.FakeHeader})
|
||||
|
||||
err := watcher.Execute()
|
||||
err := watcher.Execute()
|
||||
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(fakeTransformer.executeWasCalled).To(BeFalse())
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(fakeTransformer.ExecuteWasCalled).To(BeTrue())
|
||||
})
|
||||
|
||||
It("returns an error if transformer returns an error", func() {
|
||||
fakeTransformer = &mocks.MockTransformer{ExecuteError: errors.New("Something bad happened")}
|
||||
watcher.Transformers = []shared2.Transformer{fakeTransformer}
|
||||
repository.SetMissingHeaders([]core.Header{fakes.FakeHeader})
|
||||
|
||||
err := watcher.Execute()
|
||||
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(fakeTransformer.ExecuteWasCalled).To(BeFalse())
|
||||
})
|
||||
|
||||
It("passes only relevant logs to each transformer", func() {
|
||||
transformerA := &mocks.MockTransformer{}
|
||||
transformerB := &mocks.MockTransformer{}
|
||||
|
||||
configA := shared2.TransformerConfig{TransformerName: "transformerA",
|
||||
ContractAddresses: []string{"0x000000000000000000000000000000000000000A"},
|
||||
Topic: "0xA"}
|
||||
configB := shared2.TransformerConfig{TransformerName: "transformerB",
|
||||
ContractAddresses: []string{"0x000000000000000000000000000000000000000b"},
|
||||
Topic: "0xB"}
|
||||
|
||||
transformerA.SetTransformerConfig(configA)
|
||||
transformerB.SetTransformerConfig(configB)
|
||||
|
||||
logA := types.Log{Address: common.HexToAddress("0xA"),
|
||||
Topics: []common.Hash{common.HexToHash("0xA")}}
|
||||
logB := types.Log{Address: common.HexToAddress("0xB"),
|
||||
Topics: []common.Hash{common.HexToHash("0xB")}}
|
||||
mockBlockChain.SetGetEthLogsWithCustomQueryReturnLogs([]types.Log{logA, logB})
|
||||
|
||||
repository.SetMissingHeaders([]core.Header{fakes.FakeHeader})
|
||||
watcher = shared.NewWatcher(db, &mockBlockChain)
|
||||
watcher.AddTransformers([]shared2.TransformerInitializer{
|
||||
transformerA.FakeTransformerInitializer, transformerB.FakeTransformerInitializer})
|
||||
|
||||
err := watcher.Execute()
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(transformerA.PassedLogs).To(Equal([]types.Log{logA}))
|
||||
Expect(transformerB.PassedLogs).To(Equal([]types.Log{logB}))
|
||||
})
|
||||
|
||||
Describe("uses the LogFetcher correctly:", func() {
|
||||
BeforeEach(func() {
|
||||
repository.SetMissingHeaders([]core.Header{fakes.FakeHeader})
|
||||
})
|
||||
|
||||
It("fetches logs", func() {
|
||||
err := watcher.Execute()
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
fakeHash := common.HexToHash(fakes.FakeHeader.Hash)
|
||||
mockBlockChain.AssertGetEthLogsWithCustomQueryCalledWith(ethereum.FilterQuery{
|
||||
BlockHash: &fakeHash,
|
||||
Addresses: nil,
|
||||
Topics: [][]common.Hash{nil},
|
||||
})
|
||||
})
|
||||
|
||||
It("propagates log fetcher errors", func() {
|
||||
fetcherError := errors.New("FetcherError")
|
||||
mockBlockChain.SetGetEthLogsWithCustomQueryErr(fetcherError)
|
||||
|
||||
err := watcher.Execute()
|
||||
Expect(err).To(MatchError(fetcherError))
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@ -3,7 +3,6 @@ package repositories
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
)
|
||||
@ -51,6 +50,18 @@ func (repository HeaderRepository) MissingBlockNumbers(startingBlockNumber, endi
|
||||
return numbers
|
||||
}
|
||||
|
||||
func (repository HeaderRepository) HeaderExists(blockNumber int64) (bool, error) {
|
||||
_, err := repository.GetHeader(blockNumber)
|
||||
if err != nil {
|
||||
if headerDoesNotExist(err) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func headerMustBeReplaced(hash string, header core.Header) bool {
|
||||
return hash != header.Hash
|
||||
}
|
||||
|
||||
@ -19,27 +19,31 @@ var _ = Describe("Block header repository", func() {
|
||||
rawHeader []byte
|
||||
err error
|
||||
timestamp string
|
||||
node core.Node
|
||||
db *postgres.DB
|
||||
repo repositories.HeaderRepository
|
||||
header core.Header
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
rawHeader, err = json.Marshal(types.Header{})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
timestamp = big.NewInt(123456789).String()
|
||||
|
||||
node = core.Node{ID: "Fingerprint"}
|
||||
db = test_config.NewTestDB(node)
|
||||
test_config.CleanTestDB(db)
|
||||
repo = repositories.NewHeaderRepository(db)
|
||||
header = core.Header{
|
||||
BlockNumber: 100,
|
||||
Hash: common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
|
||||
Raw: rawHeader,
|
||||
Timestamp: timestamp,
|
||||
}
|
||||
})
|
||||
|
||||
Describe("creating or updating a header", func() {
|
||||
It("adds a header", func() {
|
||||
node := core.Node{}
|
||||
db := test_config.NewTestDB(node)
|
||||
test_config.CleanTestDB(db)
|
||||
repo := repositories.NewHeaderRepository(db)
|
||||
header := core.Header{
|
||||
BlockNumber: 100,
|
||||
Hash: common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
|
||||
Raw: rawHeader,
|
||||
Timestamp: timestamp,
|
||||
}
|
||||
|
||||
_, err := repo.CreateOrUpdateHeader(header)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
@ -53,16 +57,6 @@ var _ = Describe("Block header repository", func() {
|
||||
})
|
||||
|
||||
It("adds node data to header", func() {
|
||||
node := core.Node{ID: "EthNodeFingerprint"}
|
||||
db := test_config.NewTestDB(node)
|
||||
test_config.CleanTestDB(db)
|
||||
repo := repositories.NewHeaderRepository(db)
|
||||
header := core.Header{
|
||||
BlockNumber: 100,
|
||||
Raw: rawHeader,
|
||||
Timestamp: timestamp,
|
||||
}
|
||||
|
||||
_, err := repo.CreateOrUpdateHeader(header)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
@ -77,17 +71,6 @@ var _ = Describe("Block header repository", func() {
|
||||
})
|
||||
|
||||
It("returns valid header exists error if attempting duplicate headers", func() {
|
||||
node := core.Node{}
|
||||
db := test_config.NewTestDB(node)
|
||||
test_config.CleanTestDB(db)
|
||||
repo := repositories.NewHeaderRepository(db)
|
||||
header := core.Header{
|
||||
BlockNumber: 100,
|
||||
Hash: common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
|
||||
Raw: rawHeader,
|
||||
Timestamp: timestamp,
|
||||
}
|
||||
|
||||
_, err := repo.CreateOrUpdateHeader(header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
@ -102,16 +85,6 @@ var _ = Describe("Block header repository", func() {
|
||||
})
|
||||
|
||||
It("replaces header if hash is different", func() {
|
||||
node := core.Node{}
|
||||
db := test_config.NewTestDB(node)
|
||||
test_config.CleanTestDB(db)
|
||||
repo := repositories.NewHeaderRepository(db)
|
||||
header := core.Header{
|
||||
BlockNumber: 100,
|
||||
Hash: common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
|
||||
Raw: rawHeader,
|
||||
Timestamp: timestamp,
|
||||
}
|
||||
_, err := repo.CreateOrUpdateHeader(header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
headerTwo := core.Header{
|
||||
@ -132,16 +105,6 @@ var _ = Describe("Block header repository", func() {
|
||||
})
|
||||
|
||||
It("does not replace header if node fingerprint is different", func() {
|
||||
node := core.Node{ID: "Fingerprint"}
|
||||
db := test_config.NewTestDB(node)
|
||||
test_config.CleanTestDB(db)
|
||||
repo := repositories.NewHeaderRepository(db)
|
||||
header := core.Header{
|
||||
BlockNumber: 100,
|
||||
Hash: common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
|
||||
Raw: rawHeader,
|
||||
Timestamp: timestamp,
|
||||
}
|
||||
_, err := repo.CreateOrUpdateHeader(header)
|
||||
nodeTwo := core.Node{ID: "FingerprintTwo"}
|
||||
dbTwo, err := postgres.NewDB(test_config.DBConfig, nodeTwo)
|
||||
@ -164,15 +127,6 @@ var _ = Describe("Block header repository", func() {
|
||||
})
|
||||
|
||||
It("only replaces header with matching node fingerprint", func() {
|
||||
node := core.Node{ID: "Fingerprint"}
|
||||
db := test_config.NewTestDB(node)
|
||||
repo := repositories.NewHeaderRepository(db)
|
||||
header := core.Header{
|
||||
BlockNumber: 100,
|
||||
Hash: common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
|
||||
Raw: rawHeader,
|
||||
Timestamp: timestamp,
|
||||
}
|
||||
_, err := repo.CreateOrUpdateHeader(header)
|
||||
nodeTwo := core.Node{ID: "FingerprintTwo"}
|
||||
dbTwo, err := postgres.NewDB(test_config.DBConfig, nodeTwo)
|
||||
@ -208,16 +162,6 @@ var _ = Describe("Block header repository", func() {
|
||||
|
||||
Describe("Getting a header", func() {
|
||||
It("returns header if it exists", func() {
|
||||
node := core.Node{}
|
||||
db := test_config.NewTestDB(node)
|
||||
test_config.CleanTestDB(db)
|
||||
repo := repositories.NewHeaderRepository(db)
|
||||
header := core.Header{
|
||||
BlockNumber: 100,
|
||||
Hash: common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
|
||||
Raw: rawHeader,
|
||||
Timestamp: timestamp,
|
||||
}
|
||||
_, err := repo.CreateOrUpdateHeader(header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
@ -230,16 +174,6 @@ var _ = Describe("Block header repository", func() {
|
||||
})
|
||||
|
||||
It("does not return header for a different node fingerprint", func() {
|
||||
node := core.Node{}
|
||||
db := test_config.NewTestDB(node)
|
||||
test_config.CleanTestDB(db)
|
||||
repo := repositories.NewHeaderRepository(db)
|
||||
header := core.Header{
|
||||
BlockNumber: 100,
|
||||
Hash: common.BytesToHash(rawHeader).Hex(),
|
||||
Raw: rawHeader,
|
||||
Timestamp: timestamp,
|
||||
}
|
||||
_, err := repo.CreateOrUpdateHeader(header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
nodeTwo := core.Node{ID: "NodeFingerprintTwo"}
|
||||
@ -256,25 +190,26 @@ var _ = Describe("Block header repository", func() {
|
||||
|
||||
Describe("Getting missing headers", func() {
|
||||
It("returns block numbers for headers not in the database", func() {
|
||||
node := core.Node{}
|
||||
db := test_config.NewTestDB(node)
|
||||
test_config.CleanTestDB(db)
|
||||
repo := repositories.NewHeaderRepository(db)
|
||||
repo.CreateOrUpdateHeader(core.Header{
|
||||
_, err = repo.CreateOrUpdateHeader(core.Header{
|
||||
BlockNumber: 1,
|
||||
Raw: rawHeader,
|
||||
Timestamp: timestamp,
|
||||
})
|
||||
repo.CreateOrUpdateHeader(core.Header{
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
_, err = repo.CreateOrUpdateHeader(core.Header{
|
||||
BlockNumber: 3,
|
||||
Raw: rawHeader,
|
||||
Timestamp: timestamp,
|
||||
})
|
||||
repo.CreateOrUpdateHeader(core.Header{
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
_, err = repo.CreateOrUpdateHeader(core.Header{
|
||||
BlockNumber: 5,
|
||||
Raw: rawHeader,
|
||||
Timestamp: timestamp,
|
||||
})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
missingBlockNumbers := repo.MissingBlockNumbers(1, 5, node.ID)
|
||||
|
||||
@ -282,25 +217,27 @@ var _ = Describe("Block header repository", func() {
|
||||
})
|
||||
|
||||
It("does not count headers created by a different node fingerprint", func() {
|
||||
node := core.Node{ID: "NodeFingerprint"}
|
||||
db := test_config.NewTestDB(node)
|
||||
test_config.CleanTestDB(db)
|
||||
repo := repositories.NewHeaderRepository(db)
|
||||
repo.CreateOrUpdateHeader(core.Header{
|
||||
_, err = repo.CreateOrUpdateHeader(core.Header{
|
||||
BlockNumber: 1,
|
||||
Raw: rawHeader,
|
||||
Timestamp: timestamp,
|
||||
})
|
||||
repo.CreateOrUpdateHeader(core.Header{
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
_, err = repo.CreateOrUpdateHeader(core.Header{
|
||||
BlockNumber: 3,
|
||||
Raw: rawHeader,
|
||||
Timestamp: timestamp,
|
||||
})
|
||||
repo.CreateOrUpdateHeader(core.Header{
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
_, err = repo.CreateOrUpdateHeader(core.Header{
|
||||
BlockNumber: 5,
|
||||
Raw: rawHeader,
|
||||
Timestamp: timestamp,
|
||||
})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
nodeTwo := core.Node{ID: "NodeFingerprintTwo"}
|
||||
dbTwo, err := postgres.NewDB(test_config.DBConfig, nodeTwo)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
@ -311,4 +248,21 @@ var _ = Describe("Block header repository", func() {
|
||||
Expect(missingBlockNumbers).To(ConsistOf([]int64{1, 2, 3, 4, 5}))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("HeaderExists", func() {
|
||||
It("returns true if the header record exists", func() {
|
||||
_, err = repo.CreateOrUpdateHeader(header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
result, err := repo.HeaderExists(header.BlockNumber)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(result).To(BeTrue())
|
||||
})
|
||||
|
||||
It("returns false if the header record doesn't exist", func() {
|
||||
result, err := repo.HeaderExists(1)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(result).To(BeFalse())
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@ -41,6 +41,7 @@ type HeaderRepository interface {
|
||||
CreateOrUpdateHeader(header core.Header) (int64, error)
|
||||
GetHeader(blockNumber int64) (core.Header, error)
|
||||
MissingBlockNumbers(startingBlockNumber, endingBlockNumber int64, nodeID string) []int64
|
||||
HeaderExists(blockNumber int64) (bool, error)
|
||||
}
|
||||
|
||||
type LogRepository interface {
|
||||
|
||||
@ -12,6 +12,7 @@ type MockHeaderRepository struct {
|
||||
createOrUpdateHeaderPassedBlockNumbers []int64
|
||||
createOrUpdateHeaderReturnID int64
|
||||
missingBlockNumbers []int64
|
||||
headerExists bool
|
||||
}
|
||||
|
||||
func NewMockHeaderRepository() *MockHeaderRepository {
|
||||
@ -44,6 +45,14 @@ func (repository *MockHeaderRepository) MissingBlockNumbers(startingBlockNumber,
|
||||
return repository.missingBlockNumbers
|
||||
}
|
||||
|
||||
func (repository *MockHeaderRepository) HeaderExists(blockNumber int64) (bool, error) {
|
||||
return repository.headerExists, nil
|
||||
}
|
||||
|
||||
func (repository *MockHeaderRepository) SetHeaderExists(headerExists bool) {
|
||||
repository.headerExists = headerExists
|
||||
}
|
||||
|
||||
func (repository *MockHeaderRepository) AssertCreateOrUpdateHeaderCallCountAndPassedBlockNumbers(times int, blockNumbers []int64) {
|
||||
Expect(repository.createOrUpdateHeaderCallCount).To(Equal(times))
|
||||
Expect(repository.createOrUpdateHeaderPassedBlockNumbers).To(Equal(blockNumbers))
|
||||
|
||||
@ -10,6 +10,11 @@ import (
|
||||
func PopulateMissingBlocks(blockchain core.BlockChain, blockRepository datastore.BlockRepository, startingBlockNumber int64) int {
|
||||
lastBlock := blockchain.LastBlock().Int64()
|
||||
blockRange := blockRepository.MissingBlockNumbers(startingBlockNumber, lastBlock, blockchain.Node().ID)
|
||||
|
||||
if len(blockRange) == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
log.Printf("Backfilling %d blocks\n\n", len(blockRange))
|
||||
RetrieveAndUpdateBlocks(blockchain, blockRepository, blockRange)
|
||||
return len(blockRange)
|
||||
|
||||
@ -10,13 +10,20 @@ import (
|
||||
|
||||
func PopulateMissingHeaders(blockchain core.BlockChain, headerRepository datastore.HeaderRepository, startingBlockNumber int64) (int, error) {
|
||||
lastBlock := blockchain.LastBlock().Int64()
|
||||
blockRange := headerRepository.MissingBlockNumbers(startingBlockNumber, lastBlock, blockchain.Node().ID)
|
||||
log.Printf("Backfilling %d blocks\n\n", len(blockRange))
|
||||
_, err := RetrieveAndUpdateHeaders(blockchain, headerRepository, blockRange)
|
||||
headerAlreadyExists, err := headerRepository.HeaderExists(lastBlock)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
} else if headerAlreadyExists {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
blockNumbers := headerRepository.MissingBlockNumbers(startingBlockNumber, lastBlock, blockchain.Node().ID)
|
||||
log.Printf("Backfilling %d blocks\n\n", len(blockNumbers))
|
||||
_, err = RetrieveAndUpdateHeaders(blockchain, headerRepository, blockNumbers)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return len(blockRange), nil
|
||||
return len(blockNumbers), nil
|
||||
}
|
||||
|
||||
func RetrieveAndUpdateHeaders(chain core.BlockChain, headerRepository datastore.HeaderRepository, blockNumbers []int64) (int, error) {
|
||||
|
||||
@ -39,4 +39,14 @@ var _ = Describe("Populating headers", func() {
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
headerRepository.AssertCreateOrUpdateHeaderCallCountAndPassedBlockNumbers(1, []int64{2})
|
||||
})
|
||||
|
||||
It("returns early if the db is already synced up to the head of the chain", func() {
|
||||
blockChain := fakes.NewMockBlockChain()
|
||||
blockChain.SetLastBlock(big.NewInt(2))
|
||||
headerRepository.SetHeaderExists(true)
|
||||
headersAdded, err := history.PopulateMissingHeaders(blockChain, headerRepository, 2)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(headersAdded).To(Equal(0))
|
||||
})
|
||||
})
|
||||
|
||||
@ -25,7 +25,7 @@ The transformer process for each of these different log types is the same, excep
|
||||
## Creating a Transformer
|
||||
|
||||
1. Pull an example event (from kovan / ganache etc.)
|
||||
1. Add event sig to [`constants.go`](./shared/constants.go)
|
||||
1. Add event & method sig, contract address, `checked_headers` column name, and label to relevant files in [`constants`](./shared/constants)
|
||||
1. Write a test for the event sig in [`event_signature_generator_test.go`](./shared/event_signature_generator_test.go)
|
||||
1. Create DB table (using [`create_migration`](../../scripts/create_migration))
|
||||
1. Create columns in `checked_headers` in the _same_ migration
|
||||
@ -36,23 +36,23 @@ The transformer process for each of these different log types is the same, excep
|
||||
1. Write repository + repository tests
|
||||
1. Create converter + repository mocks
|
||||
1. Create an config object [`shared.TransformerConfig`](./shared/transformer.go) in `config.go`
|
||||
1. Create transformer + transformer tests
|
||||
1. Wire up transformer in [`transformers.go`](./transformers.go)
|
||||
1. Wire up transformer in [`transformers.go`](./transformers.go), remembering to add it to `TransformerInitializers()`
|
||||
1. Wire up transformer in [`continuousLogSync.go`](../../cmd/continuousLogSync.go)
|
||||
1. Manually trigger an event and check that it gets persisted to postgres
|
||||
1. Create an integration test for the shiny new transformer in [`integration_tests`](./integration_tests)
|
||||
|
||||
**Fetching Logs**
|
||||
|
||||
1. Generate an example raw log event, by either:
|
||||
|
||||
- Pulling the log directly from the Kovan deployment ([constants.go](https://github.com/8thlight/maker-vulcanizedb/blob/master/pkg/transformers/shared/constants.go)).
|
||||
- Pulling the log directly from the Kovan deployment ([address.go](https://github.com/8thlight/maker-vulcanizedb/blob/master/pkg/transformers/shared/constants/address.go)).
|
||||
- Deploying the contract to a local chain and emiting the event manually.
|
||||
|
||||
1. Fetch the logs from the chain based on the example event's topic zero:
|
||||
|
||||
- The topic zero is based on the keccak-256 hash of the log event's method signature. These are located in [`pkg/transformers/shared/constants.go`](./shared/constants.go).
|
||||
- Most transformers use `shared.LogFetcher` to fetch all logs that match the given topic zero for that log event.
|
||||
- Since there are multiple price feed contract address that all use the same `LogValue` event, we have a special implementation of a fetcher specifically for price feeds that can query using all of the contract addresses at once, thus only needing to make one call to the blockchain.
|
||||
- The topic zero is based on the keccak-256 hash of the log event's method signature. These are located in [`pkg/transformers/shared/constants/signature.go`](./shared/constants/signature.go).
|
||||
- Fetching is done in batch from the [`watcher`](../../libraries/shared/watcher.go).
|
||||
- The logs are then chunked up by the [`chunker`](./shared/log_chunker.go) before being delegated to each transformer.
|
||||
|
||||
**Coverting logs**
|
||||
|
||||
@ -105,8 +105,7 @@ The transformer process for each of these different log types is the same, excep
|
||||
- The `checked_headers` table allows us to keep track of which headers have been checked for a given log type.
|
||||
- To create a new migration file: `./scripts/create_migration create_flop_kick`
|
||||
- See `db/migrations/1536942529_create_flop_kick.up.sql`.
|
||||
- The specific log event tables are all created in the `maker`
|
||||
schema.
|
||||
- The specific log event tables are all created in the `maker` schema.
|
||||
- There is a one-many association between `headers` and the log
|
||||
event tables. This is so that if a header is removed due to a reorg, the associated log event records are also removed.
|
||||
- To run the migrations: `make migrate HOST=local_host PORT=5432 NAME=vulcanize_private`
|
||||
@ -120,7 +119,7 @@ The transformer process for each of these different log types is the same, excep
|
||||
|
||||
**Wire each component up in the transformer**
|
||||
|
||||
- We use a TransformerInitializer struct for each transformer so that we can inject ethRPC and postgresDB connections as well as configuration data (including the contract address, block range, etc.) into the transformer. The TransformerInitializer interface is defined in `pkg/transformers/shared/transformer.go`.
|
||||
- We use a [`TransformerInitializer`](./shared/transformer.go) struct for each transformer so that we can inject ethRPC and postgresDB connections as well as configuration data (including the contract address, block range, etc.) into the transformer.
|
||||
- See any of `pkg/transformers/flop_kick/transformer.go`
|
||||
- All of the transformers are then initialized in `pkg/transformers/transformers.go` with their configuration.
|
||||
- The transformers can be executed by using the `continuousLogSync` command, which can be configured to run specific transformers or all transformers.
|
||||
|
||||
@ -16,7 +16,6 @@ package bite
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
|
||||
@ -65,7 +64,3 @@ func (repository BiteRepository) Create(headerID int64, models []interface{}) er
|
||||
func (repository BiteRepository) MarkHeaderChecked(headerID int64) error {
|
||||
return shared.MarkHeaderChecked(headerID, repository.db, constants.BiteChecked)
|
||||
}
|
||||
|
||||
func (repository BiteRepository) MissingHeaders(startingBlockNumber int64, endingBlockNumber int64) ([]core.Header, error) {
|
||||
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.BiteChecked)
|
||||
}
|
||||
|
||||
@ -86,14 +86,4 @@ var _ = Describe("Bite repository", func() {
|
||||
|
||||
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
|
||||
})
|
||||
|
||||
Describe("MissingHeaders", func() {
|
||||
|
||||
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
|
||||
Repository: &biteRepository,
|
||||
RepositoryTwo: &bite.BiteRepository{},
|
||||
}
|
||||
|
||||
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
|
||||
})
|
||||
})
|
||||
|
||||
@ -16,7 +16,6 @@ package chop_lump
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
|
||||
@ -62,10 +61,6 @@ func (repository CatFileChopLumpRepository) MarkHeaderChecked(headerID int64) er
|
||||
return shared.MarkHeaderChecked(headerID, repository.db, constants.CatFileChopLumpChecked)
|
||||
}
|
||||
|
||||
func (repository CatFileChopLumpRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
|
||||
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.CatFileChopLumpChecked)
|
||||
}
|
||||
|
||||
func (repository *CatFileChopLumpRepository) SetDB(db *postgres.DB) {
|
||||
repository.db = db
|
||||
}
|
||||
|
||||
@ -98,13 +98,4 @@ var _ = Describe("Cat file chop lump repository", func() {
|
||||
|
||||
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
|
||||
})
|
||||
|
||||
Describe("MissingHeaders", func() {
|
||||
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
|
||||
Repository: &catFileRepository,
|
||||
RepositoryTwo: &chop_lump.CatFileChopLumpRepository{},
|
||||
}
|
||||
|
||||
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
|
||||
})
|
||||
})
|
||||
|
||||
@ -16,7 +16,6 @@ package flip
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
|
||||
@ -61,10 +60,6 @@ func (repository CatFileFlipRepository) MarkHeaderChecked(headerID int64) error
|
||||
return shared.MarkHeaderChecked(headerID, repository.db, constants.CatFileFlipChecked)
|
||||
}
|
||||
|
||||
func (repository CatFileFlipRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
|
||||
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.CatFileFlipChecked)
|
||||
}
|
||||
|
||||
func (repository *CatFileFlipRepository) SetDB(db *postgres.DB) {
|
||||
repository.db = db
|
||||
}
|
||||
|
||||
@ -83,12 +83,4 @@ var _ = Describe("Cat file flip repository", func() {
|
||||
|
||||
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
|
||||
})
|
||||
|
||||
Describe("MissingHeaders", func() {
|
||||
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
|
||||
Repository: &catFileFlipRepository,
|
||||
RepositoryTwo: &flip.CatFileFlipRepository{},
|
||||
}
|
||||
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
|
||||
})
|
||||
})
|
||||
|
||||
@ -16,7 +16,6 @@ package pit_vow
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
|
||||
@ -61,10 +60,6 @@ func (repository CatFilePitVowRepository) MarkHeaderChecked(headerID int64) erro
|
||||
return shared.MarkHeaderChecked(headerID, repository.db, constants.CatFilePitVowChecked)
|
||||
}
|
||||
|
||||
func (repository CatFilePitVowRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
|
||||
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.CatFilePitVowChecked)
|
||||
}
|
||||
|
||||
func (repository *CatFilePitVowRepository) SetDB(db *postgres.DB) {
|
||||
repository.db = db
|
||||
}
|
||||
|
||||
@ -81,13 +81,4 @@ var _ = Describe("Cat file pit vow repository", func() {
|
||||
}
|
||||
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
|
||||
})
|
||||
|
||||
Describe("MissingHeaders", func() {
|
||||
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
|
||||
Repository: &catFilePitVowRepository,
|
||||
RepositoryTwo: &pit_vow.CatFilePitVowRepository{},
|
||||
}
|
||||
|
||||
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
|
||||
})
|
||||
})
|
||||
|
||||
@ -16,7 +16,6 @@ package deal
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
|
||||
@ -62,10 +61,6 @@ func (repository DealRepository) MarkHeaderChecked(headerID int64) error {
|
||||
return shared.MarkHeaderChecked(headerID, repository.db, constants.DealChecked)
|
||||
}
|
||||
|
||||
func (repository DealRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
|
||||
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DealChecked)
|
||||
}
|
||||
|
||||
func (repository *DealRepository) SetDB(db *postgres.DB) {
|
||||
repository.db = db
|
||||
}
|
||||
|
||||
@ -84,13 +84,4 @@ var _ = Describe("Deal Repository", func() {
|
||||
|
||||
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
|
||||
})
|
||||
|
||||
Describe("MissingHeaders", func() {
|
||||
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
|
||||
Repository: &dealRepository,
|
||||
RepositoryTwo: &deal.DealRepository{},
|
||||
}
|
||||
|
||||
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
|
||||
})
|
||||
})
|
||||
|
||||
@ -16,7 +16,6 @@ package dent
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
|
||||
@ -67,10 +66,6 @@ func (repository DentRepository) MarkHeaderChecked(headerId int64) error {
|
||||
return shared.MarkHeaderChecked(headerId, repository.db, constants.DentChecked)
|
||||
}
|
||||
|
||||
func (repository DentRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
|
||||
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DentChecked)
|
||||
}
|
||||
|
||||
func (repository *DentRepository) SetDB(db *postgres.DB) {
|
||||
repository.db = db
|
||||
}
|
||||
|
||||
@ -92,13 +92,4 @@ var _ = Describe("Dent Repository", func() {
|
||||
|
||||
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
|
||||
})
|
||||
|
||||
Describe("MissingHeaders", func() {
|
||||
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
|
||||
Repository: &dentRepository,
|
||||
RepositoryTwo: &dent.DentRepository{},
|
||||
}
|
||||
|
||||
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
|
||||
})
|
||||
})
|
||||
|
||||
@ -16,7 +16,6 @@ package drip_drip
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
|
||||
@ -61,10 +60,6 @@ func (repository DripDripRepository) MarkHeaderChecked(headerID int64) error {
|
||||
return shared.MarkHeaderChecked(headerID, repository.db, constants.DripDripChecked)
|
||||
}
|
||||
|
||||
func (repository DripDripRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
|
||||
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DripDripChecked)
|
||||
}
|
||||
|
||||
func (repository *DripDripRepository) SetDB(db *postgres.DB) {
|
||||
repository.db = db
|
||||
}
|
||||
|
||||
@ -80,13 +80,4 @@ var _ = Describe("Drip drip repository", func() {
|
||||
|
||||
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
|
||||
})
|
||||
|
||||
Describe("MissingHeaders", func() {
|
||||
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
|
||||
Repository: &dripDripRepository,
|
||||
RepositoryTwo: &drip_drip.DripDripRepository{},
|
||||
}
|
||||
|
||||
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
|
||||
})
|
||||
})
|
||||
|
||||
@ -16,7 +16,6 @@ package ilk
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
|
||||
@ -64,10 +63,6 @@ func (repository DripFileIlkRepository) MarkHeaderChecked(headerID int64) error
|
||||
return shared.MarkHeaderChecked(headerID, repository.db, constants.DripFileIlkChecked)
|
||||
}
|
||||
|
||||
func (repository DripFileIlkRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
|
||||
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DripFileIlkChecked)
|
||||
}
|
||||
|
||||
func (repository *DripFileIlkRepository) SetDB(db *postgres.DB) {
|
||||
repository.db = db
|
||||
}
|
||||
|
||||
@ -82,13 +82,4 @@ var _ = Describe("Drip file ilk repository", func() {
|
||||
|
||||
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
|
||||
})
|
||||
|
||||
Describe("MissingHeaders", func() {
|
||||
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
|
||||
Repository: &dripFileIlkRepository,
|
||||
RepositoryTwo: &ilk.DripFileIlkRepository{},
|
||||
}
|
||||
|
||||
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
|
||||
})
|
||||
})
|
||||
|
||||
@ -16,7 +16,6 @@ package repo
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
|
||||
@ -64,10 +63,6 @@ func (repository DripFileRepoRepository) MarkHeaderChecked(headerID int64) error
|
||||
return shared.MarkHeaderChecked(headerID, repository.db, constants.DripFileRepoChecked)
|
||||
}
|
||||
|
||||
func (repository DripFileRepoRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
|
||||
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DripFileRepoChecked)
|
||||
}
|
||||
|
||||
func (repository *DripFileRepoRepository) SetDB(db *postgres.DB) {
|
||||
repository.db = db
|
||||
}
|
||||
|
||||
@ -82,13 +82,4 @@ var _ = Describe("Drip file repo repository", func() {
|
||||
|
||||
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
|
||||
})
|
||||
|
||||
Describe("MissingHeaders", func() {
|
||||
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
|
||||
Repository: &dripFileRepoRepository,
|
||||
RepositoryTwo: &repo.DripFileRepoRepository{},
|
||||
}
|
||||
|
||||
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
|
||||
})
|
||||
})
|
||||
|
||||
@ -16,7 +16,6 @@ package vow
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
|
||||
@ -63,10 +62,6 @@ func (repository DripFileVowRepository) MarkHeaderChecked(headerID int64) error
|
||||
return shared.MarkHeaderChecked(headerID, repository.db, constants.DripFileVowChecked)
|
||||
}
|
||||
|
||||
func (repository DripFileVowRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
|
||||
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DripFileVowChecked)
|
||||
}
|
||||
|
||||
func (repository *DripFileVowRepository) SetDB(db *postgres.DB) {
|
||||
repository.db = db
|
||||
}
|
||||
|
||||
@ -74,15 +74,6 @@ var _ = Describe("Drip file vow repository", func() {
|
||||
})
|
||||
|
||||
Describe("MarkHeaderChecked", func() {
|
||||
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
|
||||
Repository: &dripFileVowRepository,
|
||||
RepositoryTwo: &vow.DripFileVowRepository{},
|
||||
}
|
||||
|
||||
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
|
||||
})
|
||||
|
||||
Describe("MissingHeaders", func() {
|
||||
inputs := shared_behaviors.MarkedHeaderCheckedBehaviorInputs{
|
||||
CheckedHeaderColumnName: constants.DripFileVowChecked,
|
||||
Repository: &dripFileVowRepository,
|
||||
|
||||
@ -15,10 +15,9 @@
|
||||
package factories
|
||||
|
||||
import (
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
|
||||
@ -28,58 +27,44 @@ type LogNoteTransformer struct {
|
||||
Config shared.TransformerConfig
|
||||
Converter LogNoteConverter
|
||||
Repository Repository
|
||||
Fetcher shared.SettableLogFetcher
|
||||
}
|
||||
|
||||
func (transformer LogNoteTransformer) NewLogNoteTransformer(db *postgres.DB, bc core.BlockChain) shared.Transformer {
|
||||
func (transformer LogNoteTransformer) NewLogNoteTransformer(db *postgres.DB) shared.Transformer {
|
||||
transformer.Repository.SetDB(db)
|
||||
transformer.Fetcher.SetBC(bc)
|
||||
return transformer
|
||||
}
|
||||
|
||||
func (transformer LogNoteTransformer) Execute() error {
|
||||
func (transformer LogNoteTransformer) Execute(logs []types.Log, header core.Header) error {
|
||||
transformerName := transformer.Config.TransformerName
|
||||
missingHeaders, err := transformer.Repository.MissingHeaders(transformer.Config.StartingBlockNumber, transformer.Config.EndingBlockNumber)
|
||||
|
||||
// No matching logs, mark the header as checked for this type of logs
|
||||
if len(logs) < 1 {
|
||||
err := transformer.Repository.MarkHeaderChecked(header.Id)
|
||||
if err != nil {
|
||||
log.Printf("Error marking header as checked in %v: %v", transformerName, err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
models, err := transformer.Converter.ToModels(logs)
|
||||
if err != nil {
|
||||
log.Printf("Error fetching mising headers in %v transformer: %v", transformerName, err)
|
||||
log.Printf("Error converting logs in %v: %v", transformerName, err)
|
||||
return err
|
||||
}
|
||||
|
||||
// Grab event signature from transformer config
|
||||
// (Double-array structure required for go-ethereum FilterQuery)
|
||||
var topic = [][]common.Hash{{common.HexToHash(transformer.Config.Topic)}}
|
||||
|
||||
log.Printf("Fetching %v event logs for %d headers", transformerName, len(missingHeaders))
|
||||
for _, header := range missingHeaders {
|
||||
// Fetch the missing logs for a given header
|
||||
matchingLogs, err := transformer.Fetcher.FetchLogs(transformer.Config.ContractAddresses, topic, header)
|
||||
if err != nil {
|
||||
log.Printf("Error fetching matching logs in %v transformer: %v", transformerName, err)
|
||||
return err
|
||||
}
|
||||
|
||||
// No matching logs, mark the header as checked for this type of logs
|
||||
if len(matchingLogs) < 1 {
|
||||
err := transformer.Repository.MarkHeaderChecked(header.Id)
|
||||
if err != nil {
|
||||
log.Printf("Error marking header as checked in %v: %v", transformerName, err)
|
||||
return err
|
||||
}
|
||||
// Continue with the next header; nothing to persist
|
||||
continue
|
||||
}
|
||||
|
||||
models, err := transformer.Converter.ToModels(matchingLogs)
|
||||
if err != nil {
|
||||
log.Printf("Error converting logs in %v: %v", transformerName, err)
|
||||
return err
|
||||
}
|
||||
|
||||
err = transformer.Repository.Create(header.Id, models)
|
||||
if err != nil {
|
||||
log.Printf("Error persisting %v record: %v", transformerName, err)
|
||||
return err
|
||||
}
|
||||
err = transformer.Repository.Create(header.Id, models)
|
||||
if err != nil {
|
||||
log.Printf("Error persisting %v record: %v", transformerName, err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (transformer LogNoteTransformer) GetName() string {
|
||||
return transformer.Config.TransformerName
|
||||
}
|
||||
|
||||
func (transformer LogNoteTransformer) GetConfig() shared.TransformerConfig {
|
||||
return transformer.Config
|
||||
}
|
||||
|
||||
@ -15,7 +15,7 @@
|
||||
package factories_test
|
||||
|
||||
import (
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
@ -31,9 +31,7 @@ var _ = Describe("LogNoteTransformer", func() {
|
||||
var (
|
||||
repository mocks.MockRepository
|
||||
converter mocks.MockLogNoteConverter
|
||||
fetcher mocks.MockLogFetcher
|
||||
headerOne core.Header
|
||||
headerTwo core.Header
|
||||
transformer shared.Transformer
|
||||
model test_data.GenericModel
|
||||
config = test_data.GenericTestConfig
|
||||
@ -43,75 +41,28 @@ var _ = Describe("LogNoteTransformer", func() {
|
||||
BeforeEach(func() {
|
||||
repository = mocks.MockRepository{}
|
||||
converter = mocks.MockLogNoteConverter{}
|
||||
fetcher = mocks.MockLogFetcher{}
|
||||
transformer = factories.LogNoteTransformer{
|
||||
Config: config,
|
||||
Converter: &converter,
|
||||
Repository: &repository,
|
||||
Fetcher: &fetcher,
|
||||
}.NewLogNoteTransformer(nil, nil)
|
||||
}.NewLogNoteTransformer(nil)
|
||||
|
||||
headerOne = core.Header{Id: rand.Int63(), BlockNumber: rand.Int63()}
|
||||
headerTwo = core.Header{Id: rand.Int63(), BlockNumber: rand.Int63()}
|
||||
})
|
||||
|
||||
It("sets the blockchain and database", func() {
|
||||
Expect(fetcher.SetBcCalled).To(BeTrue())
|
||||
It("sets the database", func() {
|
||||
Expect(repository.SetDbCalled).To(BeTrue())
|
||||
})
|
||||
|
||||
It("gets missing headers for block numbers specified in config", func() {
|
||||
err := transformer.Execute()
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(repository.PassedStartingBlockNumber).To(Equal(config.StartingBlockNumber))
|
||||
Expect(repository.PassedEndingBlockNumber).To(Equal(config.EndingBlockNumber))
|
||||
})
|
||||
|
||||
It("returns error if repository returns error for missing headers", func() {
|
||||
repository.SetMissingHeadersError(fakes.FakeError)
|
||||
|
||||
err := transformer.Execute()
|
||||
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err).To(MatchError(fakes.FakeError))
|
||||
})
|
||||
|
||||
It("fetches logs for missing headers", func() {
|
||||
repository.SetMissingHeaders([]core.Header{headerOne, headerTwo})
|
||||
|
||||
err := transformer.Execute()
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(fetcher.FetchedBlocks).To(Equal([]int64{headerOne.BlockNumber, headerTwo.BlockNumber}))
|
||||
Expect(fetcher.FetchedContractAddresses).To(Equal([][]string{config.ContractAddresses, config.ContractAddresses}))
|
||||
expectedTopic := common.HexToHash(config.Topic)
|
||||
Expect(fetcher.FetchedTopics).To(Equal([][]common.Hash{{expectedTopic}}))
|
||||
})
|
||||
|
||||
It("returns error if fetcher returns error", func() {
|
||||
repository.SetMissingHeaders([]core.Header{headerOne})
|
||||
fetcher.SetFetcherError(fakes.FakeError)
|
||||
|
||||
err := transformer.Execute()
|
||||
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err).To(MatchError(fakes.FakeError))
|
||||
})
|
||||
|
||||
It("marks header checked if no logs returned", func() {
|
||||
repository.SetMissingHeaders([]core.Header{headerOne})
|
||||
|
||||
err := transformer.Execute()
|
||||
It("marks header checked if no logs are provided", func() {
|
||||
err := transformer.Execute([]types.Log{}, headerOne)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
repository.AssertMarkHeaderCheckedCalledWith(headerOne.Id)
|
||||
})
|
||||
|
||||
It("doesn't attempt to convert or persist an empty collection when there are no logs", func() {
|
||||
repository.SetMissingHeaders([]core.Header{headerOne, headerTwo})
|
||||
|
||||
err := transformer.Execute()
|
||||
err := transformer.Execute([]types.Log{}, headerOne)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(converter.ToModelsCalledCounter).To(Equal(0))
|
||||
@ -119,30 +70,23 @@ var _ = Describe("LogNoteTransformer", func() {
|
||||
})
|
||||
|
||||
It("does not call repository.MarkCheckedHeader when there are logs", func() {
|
||||
repository.SetMissingHeaders([]core.Header{headerOne})
|
||||
fetcher.SetFetchedLogs(logs)
|
||||
|
||||
err := transformer.Execute()
|
||||
err := transformer.Execute(logs, headerOne)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
repository.AssertMarkHeaderCheckedNotCalled()
|
||||
})
|
||||
|
||||
It("returns error if marking header checked returns err", func() {
|
||||
repository.SetMissingHeaders([]core.Header{headerOne})
|
||||
repository.SetMarkHeaderCheckedError(fakes.FakeError)
|
||||
|
||||
err := transformer.Execute()
|
||||
err := transformer.Execute([]types.Log{}, headerOne)
|
||||
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err).To(MatchError(fakes.FakeError))
|
||||
})
|
||||
|
||||
It("converts matching logs to models", func() {
|
||||
fetcher.SetFetchedLogs(logs)
|
||||
repository.SetMissingHeaders([]core.Header{headerOne})
|
||||
|
||||
err := transformer.Execute()
|
||||
err := transformer.Execute(logs, headerOne)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(converter.PassedLogs).To(Equal(logs))
|
||||
@ -150,20 +94,16 @@ var _ = Describe("LogNoteTransformer", func() {
|
||||
|
||||
It("returns error if converter returns error", func() {
|
||||
converter.SetConverterError(fakes.FakeError)
|
||||
fetcher.SetFetchedLogs(logs)
|
||||
repository.SetMissingHeaders([]core.Header{headerOne})
|
||||
|
||||
err := transformer.Execute()
|
||||
err := transformer.Execute(logs, headerOne)
|
||||
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err).To(MatchError(fakes.FakeError))
|
||||
})
|
||||
|
||||
It("persists the model", func() {
|
||||
fetcher.SetFetchedLogs(logs)
|
||||
repository.SetMissingHeaders([]core.Header{headerOne})
|
||||
converter.SetReturnModels([]interface{}{model})
|
||||
err := transformer.Execute()
|
||||
err := transformer.Execute(logs, headerOne)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(repository.PassedHeaderID).To(Equal(headerOne.Id))
|
||||
@ -171,11 +111,9 @@ var _ = Describe("LogNoteTransformer", func() {
|
||||
})
|
||||
|
||||
It("returns error if repository returns error for create", func() {
|
||||
fetcher.SetFetchedLogs(logs)
|
||||
repository.SetMissingHeaders([]core.Header{headerOne})
|
||||
repository.SetCreateError(fakes.FakeError)
|
||||
|
||||
err := transformer.Execute()
|
||||
err := transformer.Execute(logs, headerOne)
|
||||
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err).To(MatchError(fakes.FakeError))
|
||||
|
||||
@ -15,13 +15,11 @@
|
||||
package factories
|
||||
|
||||
import (
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
)
|
||||
|
||||
type Repository interface {
|
||||
Create(headerID int64, models []interface{}) error
|
||||
MarkHeaderChecked(headerID int64) error
|
||||
MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
|
||||
SetDB(db *postgres.DB)
|
||||
}
|
||||
|
||||
@ -15,10 +15,9 @@
|
||||
package factories
|
||||
|
||||
import (
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
|
||||
@ -28,61 +27,51 @@ type Transformer struct {
|
||||
Config shared.TransformerConfig
|
||||
Converter Converter
|
||||
Repository Repository
|
||||
Fetcher shared.SettableLogFetcher
|
||||
}
|
||||
|
||||
func (transformer Transformer) NewTransformer(db *postgres.DB, bc core.BlockChain) shared.Transformer {
|
||||
func (transformer Transformer) NewTransformer(db *postgres.DB) shared.Transformer {
|
||||
transformer.Repository.SetDB(db)
|
||||
transformer.Fetcher.SetBC(bc)
|
||||
return transformer
|
||||
}
|
||||
|
||||
func (transformer Transformer) Execute() error {
|
||||
func (transformer Transformer) Execute(logs []types.Log, header core.Header) error {
|
||||
transformerName := transformer.Config.TransformerName
|
||||
config := transformer.Config
|
||||
topics := [][]common.Hash{{common.HexToHash(config.Topic)}}
|
||||
missingHeaders, err := transformer.Repository.MissingHeaders(config.StartingBlockNumber, config.EndingBlockNumber)
|
||||
|
||||
if len(logs) < 1 {
|
||||
err := transformer.Repository.MarkHeaderChecked(header.Id)
|
||||
if err != nil {
|
||||
log.Printf("Error marking header as checked in %v: %v", transformerName, err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
entities, err := transformer.Converter.ToEntities(config.ContractAbi, logs)
|
||||
if err != nil {
|
||||
log.Printf("Error fetching missing headers in %v transformer: %v \n", transformerName, err)
|
||||
log.Printf("Error converting logs to entities in %v: %v", transformerName, err)
|
||||
return err
|
||||
}
|
||||
log.Printf("Fetching %v event logs for %d headers \n", transformerName, len(missingHeaders))
|
||||
for _, header := range missingHeaders {
|
||||
logs, err := transformer.Fetcher.FetchLogs(config.ContractAddresses, topics, header)
|
||||
if err != nil {
|
||||
log.Printf("Error fetching matching logs in %v transformer: %v", transformerName, err)
|
||||
return err
|
||||
}
|
||||
|
||||
if len(logs) < 1 {
|
||||
err = transformer.Repository.MarkHeaderChecked(header.Id)
|
||||
if err != nil {
|
||||
log.Printf("Error marking header as checked in %v: %v", transformerName, err)
|
||||
return err
|
||||
}
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
entities, err := transformer.Converter.ToEntities(config.ContractAbi, logs)
|
||||
if err != nil {
|
||||
log.Printf("Error converting logs to entities in %v: %v", transformerName, err)
|
||||
return err
|
||||
}
|
||||
|
||||
models, err := transformer.Converter.ToModels(entities)
|
||||
if err != nil {
|
||||
log.Printf("Error converting entities to models in %v: %v", transformerName, err)
|
||||
return err
|
||||
}
|
||||
|
||||
err = transformer.Repository.Create(header.Id, models)
|
||||
if err != nil {
|
||||
log.Printf("Error persisting %v record: %v", transformerName, err)
|
||||
return err
|
||||
}
|
||||
models, err := transformer.Converter.ToModels(entities)
|
||||
if err != nil {
|
||||
log.Printf("Error converting entities to models in %v: %v", transformerName, err)
|
||||
return err
|
||||
}
|
||||
|
||||
err = transformer.Repository.Create(header.Id, models)
|
||||
if err != nil {
|
||||
log.Printf("Error persisting %v record: %v", transformerName, err)
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (transformer Transformer) GetName() string {
|
||||
return transformer.Config.TransformerName
|
||||
}
|
||||
|
||||
func (transformer Transformer) GetConfig() shared.TransformerConfig {
|
||||
return transformer.Config
|
||||
}
|
||||
|
||||
@ -15,7 +15,7 @@
|
||||
package factories_test
|
||||
|
||||
import (
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
@ -30,86 +30,39 @@ import (
|
||||
var _ = Describe("Transformer", func() {
|
||||
var (
|
||||
repository mocks.MockRepository
|
||||
fetcher mocks.MockLogFetcher
|
||||
converter mocks.MockConverter
|
||||
transformer shared.Transformer
|
||||
headerOne core.Header
|
||||
headerTwo core.Header
|
||||
config = test_data.GenericTestConfig
|
||||
logs = test_data.GenericTestLogs
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
repository = mocks.MockRepository{}
|
||||
fetcher = mocks.MockLogFetcher{}
|
||||
converter = mocks.MockConverter{}
|
||||
|
||||
transformer = factories.Transformer{
|
||||
Repository: &repository,
|
||||
Fetcher: &fetcher,
|
||||
Converter: &converter,
|
||||
Config: config,
|
||||
}.NewTransformer(nil, nil)
|
||||
}.NewTransformer(nil)
|
||||
|
||||
headerOne = core.Header{Id: rand.Int63(), BlockNumber: rand.Int63()}
|
||||
headerTwo = core.Header{Id: rand.Int63(), BlockNumber: rand.Int63()}
|
||||
})
|
||||
|
||||
It("sets the blockchain and db", func() {
|
||||
Expect(fetcher.SetBcCalled).To(BeTrue())
|
||||
It("sets the db", func() {
|
||||
Expect(repository.SetDbCalled).To(BeTrue())
|
||||
})
|
||||
|
||||
It("gets missing headers for blocks in the configured range", func() {
|
||||
err := transformer.Execute()
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(repository.PassedStartingBlockNumber).To(Equal(config.StartingBlockNumber))
|
||||
Expect(repository.PassedEndingBlockNumber).To(Equal(config.EndingBlockNumber))
|
||||
})
|
||||
|
||||
It("returns an error if it fails to get missing headers", func() {
|
||||
repository.SetMissingHeadersError(fakes.FakeError)
|
||||
err := transformer.Execute()
|
||||
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err).To(MatchError(fakes.FakeError))
|
||||
})
|
||||
|
||||
It("fetches eth logs for each missing header", func() {
|
||||
repository.SetMissingHeaders([]core.Header{headerOne, headerTwo})
|
||||
expectedTopics := [][]common.Hash{{common.HexToHash(config.Topic)}}
|
||||
err := transformer.Execute()
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(fetcher.FetchedBlocks).To(Equal([]int64{headerOne.BlockNumber, headerTwo.BlockNumber}))
|
||||
Expect(fetcher.FetchedTopics).To(Equal(expectedTopics))
|
||||
Expect(fetcher.FetchedContractAddresses).To(Equal([][]string{config.ContractAddresses, config.ContractAddresses}))
|
||||
})
|
||||
|
||||
It("returns an error if fetching logs fails", func() {
|
||||
repository.SetMissingHeaders([]core.Header{headerOne})
|
||||
fetcher.SetFetcherError(fakes.FakeError)
|
||||
err := transformer.Execute()
|
||||
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err).To(MatchError(fakes.FakeError))
|
||||
})
|
||||
|
||||
It("marks header checked if no logs returned", func() {
|
||||
headerID := int64(123)
|
||||
repository.SetMissingHeaders([]core.Header{{Id: headerID}})
|
||||
|
||||
err := transformer.Execute()
|
||||
err := transformer.Execute([]types.Log{}, headerOne)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
repository.AssertMarkHeaderCheckedCalledWith(headerID)
|
||||
repository.AssertMarkHeaderCheckedCalledWith(headerOne.Id)
|
||||
})
|
||||
|
||||
It("doesn't attempt to convert or persist an empty collection when there are no logs", func() {
|
||||
repository.SetMissingHeaders([]core.Header{headerOne, headerTwo})
|
||||
|
||||
err := transformer.Execute()
|
||||
err := transformer.Execute([]types.Log{}, headerOne)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(converter.ToEntitiesCalledCounter).To(Equal(0))
|
||||
@ -118,29 +71,23 @@ var _ = Describe("Transformer", func() {
|
||||
})
|
||||
|
||||
It("does not call repository.MarkCheckedHeader when there are logs", func() {
|
||||
repository.SetMissingHeaders([]core.Header{headerOne})
|
||||
fetcher.SetFetchedLogs(logs)
|
||||
|
||||
err := transformer.Execute()
|
||||
err := transformer.Execute(logs, headerOne)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
repository.AssertMarkHeaderCheckedNotCalled()
|
||||
})
|
||||
|
||||
It("returns error if marking header checked returns err", func() {
|
||||
repository.SetMissingHeaders([]core.Header{headerOne})
|
||||
repository.SetMarkHeaderCheckedError(fakes.FakeError)
|
||||
|
||||
err := transformer.Execute()
|
||||
err := transformer.Execute([]types.Log{}, headerOne)
|
||||
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err).To(MatchError(fakes.FakeError))
|
||||
})
|
||||
|
||||
It("converts an eth log to an entity", func() {
|
||||
repository.SetMissingHeaders([]core.Header{headerOne})
|
||||
fetcher.SetFetchedLogs(logs)
|
||||
err := transformer.Execute()
|
||||
err := transformer.Execute(logs, headerOne)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(converter.ContractAbi).To(Equal(config.ContractAbi))
|
||||
@ -148,45 +95,37 @@ var _ = Describe("Transformer", func() {
|
||||
})
|
||||
|
||||
It("returns an error if converter fails", func() {
|
||||
repository.SetMissingHeaders([]core.Header{headerOne})
|
||||
fetcher.SetFetchedLogs(logs)
|
||||
converter.ToEntitiesError = fakes.FakeError
|
||||
|
||||
err := transformer.Execute()
|
||||
err := transformer.Execute(logs, headerOne)
|
||||
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err).To(MatchError(fakes.FakeError))
|
||||
})
|
||||
|
||||
It("converts an entity to a model", func() {
|
||||
repository.SetMissingHeaders([]core.Header{headerOne})
|
||||
fetcher.SetFetchedLogs(logs)
|
||||
converter.EntitiesToReturn = []interface{}{test_data.GenericEntity{}}
|
||||
|
||||
err := transformer.Execute()
|
||||
err := transformer.Execute(logs, headerOne)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(converter.EntitiesToConvert[0]).To(Equal(test_data.GenericEntity{}))
|
||||
})
|
||||
|
||||
It("returns an error if converting to models fails", func() {
|
||||
repository.SetMissingHeaders([]core.Header{headerOne})
|
||||
fetcher.SetFetchedLogs(logs)
|
||||
converter.EntitiesToReturn = []interface{}{test_data.GenericEntity{}}
|
||||
converter.ToModelsError = fakes.FakeError
|
||||
|
||||
err := transformer.Execute()
|
||||
err := transformer.Execute(logs, headerOne)
|
||||
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err).To(MatchError(fakes.FakeError))
|
||||
})
|
||||
|
||||
It("persists the record", func() {
|
||||
repository.SetMissingHeaders([]core.Header{headerOne})
|
||||
fetcher.SetFetchedLogs(logs)
|
||||
converter.ModelsToReturn = []interface{}{test_data.GenericModel{}}
|
||||
|
||||
err := transformer.Execute()
|
||||
err := transformer.Execute(logs, headerOne)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(repository.PassedHeaderID).To(Equal(headerOne.Id))
|
||||
@ -194,10 +133,8 @@ var _ = Describe("Transformer", func() {
|
||||
})
|
||||
|
||||
It("returns error if persisting the record fails", func() {
|
||||
repository.SetMissingHeaders([]core.Header{headerOne})
|
||||
fetcher.SetFetchedLogs(logs)
|
||||
repository.SetCreateError(fakes.FakeError)
|
||||
err := transformer.Execute()
|
||||
err := transformer.Execute(logs, headerOne)
|
||||
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err).To(MatchError(fakes.FakeError))
|
||||
|
||||
@ -16,7 +16,6 @@ package flap_kick
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
|
||||
@ -60,10 +59,6 @@ func (repository *FlapKickRepository) MarkHeaderChecked(headerID int64) error {
|
||||
return shared.MarkHeaderChecked(headerID, repository.db, constants.FlapKickChecked)
|
||||
}
|
||||
|
||||
func (repository FlapKickRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
|
||||
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.FlapKickChecked)
|
||||
}
|
||||
|
||||
func (repository *FlapKickRepository) SetDB(db *postgres.DB) {
|
||||
repository.db = db
|
||||
}
|
||||
|
||||
@ -87,13 +87,4 @@ var _ = Describe("Flap Kick Repository", func() {
|
||||
|
||||
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
|
||||
})
|
||||
|
||||
Describe("MissingHeaders", func() {
|
||||
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
|
||||
Repository: &flapKickRepository,
|
||||
RepositoryTwo: &flap_kick.FlapKickRepository{},
|
||||
}
|
||||
|
||||
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
|
||||
})
|
||||
})
|
||||
|
||||
@ -17,7 +17,6 @@ package flip_kick
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
|
||||
@ -60,10 +59,6 @@ func (repository FlipKickRepository) MarkHeaderChecked(headerId int64) error {
|
||||
return shared.MarkHeaderChecked(headerId, repository.db, constants.FlipKickChecked)
|
||||
}
|
||||
|
||||
func (repository FlipKickRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
|
||||
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.FlipKickChecked)
|
||||
}
|
||||
|
||||
func (repository *FlipKickRepository) SetDB(db *postgres.DB) {
|
||||
repository.db = db
|
||||
}
|
||||
|
||||
@ -87,15 +87,6 @@ var _ = Describe("FlipKick Repository", func() {
|
||||
|
||||
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
|
||||
})
|
||||
|
||||
Describe("missing headers", func() {
|
||||
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
|
||||
Repository: &flipKickRepository,
|
||||
RepositoryTwo: &flip_kick.FlipKickRepository{},
|
||||
}
|
||||
|
||||
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
|
||||
})
|
||||
})
|
||||
|
||||
func assertDBRecordCount(db *postgres.DB, dbTable string, expectedCount int) {
|
||||
|
||||
@ -16,7 +16,6 @@ package flop_kick
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
|
||||
@ -62,10 +61,6 @@ func (repository FlopKickRepository) MarkHeaderChecked(headerId int64) error {
|
||||
return shared.MarkHeaderChecked(headerId, repository.db, constants.FlopKickChecked)
|
||||
}
|
||||
|
||||
func (repository FlopKickRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
|
||||
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.FlopKickChecked)
|
||||
}
|
||||
|
||||
func (repository *FlopKickRepository) SetDB(db *postgres.DB) {
|
||||
repository.db = db
|
||||
}
|
||||
|
||||
@ -85,13 +85,4 @@ var _ = Describe("FlopRepository", func() {
|
||||
|
||||
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
|
||||
})
|
||||
|
||||
Describe("MissingHeaders", func() {
|
||||
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
|
||||
Repository: &repository,
|
||||
RepositoryTwo: &flop_kick.FlopKickRepository{},
|
||||
}
|
||||
|
||||
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
|
||||
})
|
||||
})
|
||||
|
||||
@ -17,7 +17,6 @@ package frob
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
|
||||
@ -59,10 +58,6 @@ func (repository FrobRepository) MarkHeaderChecked(headerID int64) error {
|
||||
return shared.MarkHeaderChecked(headerID, repository.db, constants.FrobChecked)
|
||||
}
|
||||
|
||||
func (repository FrobRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
|
||||
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.FrobChecked)
|
||||
}
|
||||
|
||||
func (repository *FrobRepository) SetDB(db *postgres.DB) {
|
||||
repository.db = db
|
||||
}
|
||||
|
||||
@ -86,13 +86,4 @@ var _ = Describe("Frob repository", func() {
|
||||
|
||||
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
|
||||
})
|
||||
|
||||
Describe("MissingHeaders", func() {
|
||||
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
|
||||
Repository: &frobRepository,
|
||||
RepositoryTwo: &frob.FrobRepository{},
|
||||
}
|
||||
|
||||
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
|
||||
})
|
||||
})
|
||||
|
||||
@ -45,17 +45,24 @@ var _ = Describe("Bite Transformer", func() {
|
||||
db := test_config.NewTestDB(blockChain.Node())
|
||||
test_config.CleanTestDB(db)
|
||||
|
||||
err = persistHeader(db, blockNumber, blockChain)
|
||||
header, err := persistHeader(db, blockNumber, blockChain)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
initializer := factories.Transformer{
|
||||
Config: config,
|
||||
Converter: &bite.BiteConverter{},
|
||||
Repository: &bite.BiteRepository{},
|
||||
Fetcher: &shared.Fetcher{},
|
||||
}
|
||||
transformer := initializer.NewTransformer(db, blockChain)
|
||||
err = transformer.Execute()
|
||||
transformer := initializer.NewTransformer(db)
|
||||
|
||||
fetcher := shared.NewFetcher(blockChain)
|
||||
logs, err := fetcher.FetchLogs(
|
||||
[]common.Address{common.HexToAddress(config.ContractAddresses[0])},
|
||||
[]common.Hash{common.HexToHash(config.Topic)},
|
||||
header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
err = transformer.Execute(logs, header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
var dbResult []bite.BiteModel
|
||||
|
||||
@ -15,6 +15,7 @@
|
||||
package integration_tests
|
||||
|
||||
import (
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
|
||||
"sort"
|
||||
@ -39,6 +40,7 @@ var _ = Describe("Cat File transformer", func() {
|
||||
rpcClient client.RpcClient
|
||||
err error
|
||||
ethClient *ethclient.Client
|
||||
fetcher *shared.Fetcher
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
@ -48,13 +50,15 @@ var _ = Describe("Cat File transformer", func() {
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
db = test_config.NewTestDB(blockChain.Node())
|
||||
test_config.CleanTestDB(db)
|
||||
|
||||
fetcher = shared.NewFetcher(blockChain)
|
||||
})
|
||||
|
||||
// Cat contract Kovan address: 0x2f34f22a00ee4b7a8f8bbc4eaee1658774c624e0
|
||||
It("persists a chop lump event", func() {
|
||||
// transaction: 0x98574bfba4d05c3875be10d2376e678d005dbebe9a4520363407508fd21f4014
|
||||
chopLumpBlockNumber := int64(8762253)
|
||||
err = persistHeader(db, chopLumpBlockNumber, blockChain)
|
||||
header, err := persistHeader(db, chopLumpBlockNumber, blockChain)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
config := chop_lump.CatFileChopLumpConfig
|
||||
@ -65,10 +69,16 @@ var _ = Describe("Cat File transformer", func() {
|
||||
Config: config,
|
||||
Converter: &chop_lump.CatFileChopLumpConverter{},
|
||||
Repository: &chop_lump.CatFileChopLumpRepository{},
|
||||
Fetcher: &shared.Fetcher{},
|
||||
}
|
||||
transformer := initializer.NewLogNoteTransformer(db, blockChain)
|
||||
err := transformer.Execute()
|
||||
transformer := initializer.NewLogNoteTransformer(db)
|
||||
|
||||
logs, err := fetcher.FetchLogs(
|
||||
[]common.Address{common.HexToAddress(config.ContractAddresses[0])},
|
||||
[]common.Hash{common.HexToHash(config.Topic)},
|
||||
header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
err = transformer.Execute(logs, header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
var dbResult []chop_lump.CatFileChopLumpModel
|
||||
@ -92,7 +102,7 @@ var _ = Describe("Cat File transformer", func() {
|
||||
It("persists a flip event", func() {
|
||||
// transaction: 0x44bc18fdb1a5a263db114e7879653304db3e19ceb4e4496f21bc0a76c5faccbe
|
||||
flipBlockNumber := int64(8751794)
|
||||
err = persistHeader(db, flipBlockNumber, blockChain)
|
||||
header, err := persistHeader(db, flipBlockNumber, blockChain)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
config := flip.CatFileFlipConfig
|
||||
@ -103,10 +113,17 @@ var _ = Describe("Cat File transformer", func() {
|
||||
Config: config,
|
||||
Converter: &flip.CatFileFlipConverter{},
|
||||
Repository: &flip.CatFileFlipRepository{},
|
||||
Fetcher: &shared.Fetcher{},
|
||||
}
|
||||
transformer := initializer.NewLogNoteTransformer(db, blockChain)
|
||||
err := transformer.Execute()
|
||||
|
||||
transformer := initializer.NewLogNoteTransformer(db)
|
||||
|
||||
logs, err := fetcher.FetchLogs(
|
||||
[]common.Address{common.HexToAddress(config.ContractAddresses[0])},
|
||||
[]common.Hash{common.HexToHash(config.Topic)},
|
||||
header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
err = transformer.Execute(logs, header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
var dbResult []flip.CatFileFlipModel
|
||||
@ -122,7 +139,7 @@ var _ = Describe("Cat File transformer", func() {
|
||||
It("persists a pit vow event", func() {
|
||||
// transaction: 0x44bc18fdb1a5a263db114e7879653304db3e19ceb4e4496f21bc0a76c5faccbe
|
||||
pitVowBlockNumber := int64(8751794)
|
||||
err = persistHeader(db, pitVowBlockNumber, blockChain)
|
||||
header, err := persistHeader(db, pitVowBlockNumber, blockChain)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
config := pit_vow.CatFilePitVowConfig
|
||||
@ -133,10 +150,16 @@ var _ = Describe("Cat File transformer", func() {
|
||||
Config: config,
|
||||
Converter: &pit_vow.CatFilePitVowConverter{},
|
||||
Repository: &pit_vow.CatFilePitVowRepository{},
|
||||
Fetcher: &shared.Fetcher{},
|
||||
}
|
||||
transformer := initializer.NewLogNoteTransformer(db, blockChain)
|
||||
err := transformer.Execute()
|
||||
transformer := initializer.NewLogNoteTransformer(db)
|
||||
|
||||
logs, err := fetcher.FetchLogs(
|
||||
[]common.Address{common.HexToAddress(config.ContractAddresses[0])},
|
||||
[]common.Hash{common.HexToHash(config.Topic)},
|
||||
header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
err = transformer.Execute(logs, header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
var dbResult []pit_vow.CatFilePitVowModel
|
||||
|
||||
@ -15,6 +15,7 @@
|
||||
package integration_tests
|
||||
|
||||
import (
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
@ -29,8 +30,13 @@ import (
|
||||
|
||||
var _ = Describe("Deal transformer", func() {
|
||||
var (
|
||||
db *postgres.DB
|
||||
blockChain core.BlockChain
|
||||
db *postgres.DB
|
||||
blockChain core.BlockChain
|
||||
config shared.TransformerConfig
|
||||
initializer factories.LogNoteTransformer
|
||||
fetcher *shared.Fetcher
|
||||
addresses []common.Address
|
||||
topics []common.Hash
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
@ -40,26 +46,35 @@ var _ = Describe("Deal transformer", func() {
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
db = test_config.NewTestDB(blockChain.Node())
|
||||
test_config.CleanTestDB(db)
|
||||
|
||||
config = deal.DealConfig
|
||||
|
||||
initializer = factories.LogNoteTransformer{
|
||||
Config: config,
|
||||
Converter: &deal.DealConverter{},
|
||||
Repository: &deal.DealRepository{},
|
||||
}
|
||||
|
||||
fetcher = shared.NewFetcher(blockChain)
|
||||
addresses = shared.HexStringsToAddresses(config.ContractAddresses)
|
||||
topics = []common.Hash{common.HexToHash(config.Topic)}
|
||||
|
||||
})
|
||||
|
||||
It("persists a flip deal log event", func() {
|
||||
// transaction: 0x05b5eabac2ace136f0f7e0efc61d7d42abe8e8938cc0f04fbf1a6ba545d59e58
|
||||
flipBlockNumber := int64(8958007)
|
||||
err := persistHeader(db, flipBlockNumber, blockChain)
|
||||
header, err := persistHeader(db, flipBlockNumber, blockChain)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
config := deal.DealConfig
|
||||
config.StartingBlockNumber = flipBlockNumber
|
||||
config.EndingBlockNumber = flipBlockNumber
|
||||
initializer.Config.StartingBlockNumber = flipBlockNumber
|
||||
initializer.Config.EndingBlockNumber = flipBlockNumber
|
||||
|
||||
initializer := factories.LogNoteTransformer{
|
||||
Config: config,
|
||||
Converter: &deal.DealConverter{},
|
||||
Repository: &deal.DealRepository{},
|
||||
Fetcher: &shared.Fetcher{},
|
||||
}
|
||||
transformer := initializer.NewLogNoteTransformer(db, blockChain)
|
||||
err = transformer.Execute()
|
||||
logs, err := fetcher.FetchLogs(addresses, topics, header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
transformer := initializer.NewLogNoteTransformer(db)
|
||||
err = transformer.Execute(logs, header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
var dbResult []deal.DealModel
|
||||
@ -77,21 +92,17 @@ var _ = Describe("Deal transformer", func() {
|
||||
|
||||
It("persists a flap deal log event", func() {
|
||||
flapBlockNumber := int64(9004628)
|
||||
err := persistHeader(db, flapBlockNumber, blockChain)
|
||||
header, err := persistHeader(db, flapBlockNumber, blockChain)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
config := deal.DealConfig
|
||||
config.StartingBlockNumber = flapBlockNumber
|
||||
config.EndingBlockNumber = flapBlockNumber
|
||||
initializer.Config.StartingBlockNumber = flapBlockNumber
|
||||
initializer.Config.EndingBlockNumber = flapBlockNumber
|
||||
|
||||
initializer := factories.LogNoteTransformer{
|
||||
Config: config,
|
||||
Converter: &deal.DealConverter{},
|
||||
Repository: &deal.DealRepository{},
|
||||
Fetcher: &shared.Fetcher{},
|
||||
}
|
||||
transformer := initializer.NewLogNoteTransformer(db, blockChain)
|
||||
err = transformer.Execute()
|
||||
logs, err := fetcher.FetchLogs(addresses, topics, header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
transformer := initializer.NewLogNoteTransformer(db)
|
||||
err = transformer.Execute(logs, header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
var dbResult []deal.DealModel
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
package integration_tests
|
||||
|
||||
import (
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
|
||||
@ -15,8 +16,14 @@ import (
|
||||
|
||||
var _ = Describe("Dent transformer", func() {
|
||||
var (
|
||||
db *postgres.DB
|
||||
blockChain core.BlockChain
|
||||
db *postgres.DB
|
||||
blockChain core.BlockChain
|
||||
fetcher *shared.Fetcher
|
||||
transformer shared.Transformer
|
||||
config shared.TransformerConfig
|
||||
addresses []common.Address
|
||||
topics []common.Hash
|
||||
initializer factories.LogNoteTransformer
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
@ -26,25 +33,32 @@ var _ = Describe("Dent transformer", func() {
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
db = test_config.NewTestDB(blockChain.Node())
|
||||
test_config.CleanTestDB(db)
|
||||
|
||||
config = dent.DentConfig
|
||||
addresses = shared.HexStringsToAddresses(config.ContractAddresses)
|
||||
topics = []common.Hash{common.HexToHash(config.Topic)}
|
||||
fetcher = shared.NewFetcher(blockChain)
|
||||
|
||||
initializer = factories.LogNoteTransformer{
|
||||
Config: config,
|
||||
Converter: &dent.DentConverter{},
|
||||
Repository: &dent.DentRepository{},
|
||||
}
|
||||
})
|
||||
|
||||
It("persists a flop dent log event", func() {
|
||||
blockNumber := int64(8955613)
|
||||
err := persistHeader(db, blockNumber, blockChain)
|
||||
header, err := persistHeader(db, blockNumber, blockChain)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
config := dent.DentConfig
|
||||
config.StartingBlockNumber = blockNumber
|
||||
config.EndingBlockNumber = blockNumber
|
||||
initializer.Config.StartingBlockNumber = blockNumber
|
||||
initializer.Config.EndingBlockNumber = blockNumber
|
||||
|
||||
initializer := factories.LogNoteTransformer{
|
||||
Config: config,
|
||||
Converter: &dent.DentConverter{},
|
||||
Repository: &dent.DentRepository{},
|
||||
Fetcher: &shared.Fetcher{},
|
||||
}
|
||||
transformer := initializer.NewLogNoteTransformer(db, blockChain)
|
||||
err = transformer.Execute()
|
||||
logs, err := fetcher.FetchLogs(addresses, topics, header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
transformer = initializer.NewLogNoteTransformer(db)
|
||||
err = transformer.Execute(logs, header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
var dbResult []dent.DentModel
|
||||
|
||||
@ -15,8 +15,11 @@
|
||||
package integration_tests
|
||||
|
||||
import (
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
|
||||
|
||||
@ -25,32 +28,43 @@ import (
|
||||
)
|
||||
|
||||
var _ = Describe("DripDrip Transformer", func() {
|
||||
var (
|
||||
db *postgres.DB
|
||||
blockChain core.BlockChain
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
rpcClient, ethClient, err := getClients(ipc)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
blockChain, err = getBlockChain(rpcClient, ethClient)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
db = test_config.NewTestDB(blockChain.Node())
|
||||
test_config.CleanTestDB(db)
|
||||
})
|
||||
It("transforms DripDrip log events", func() {
|
||||
blockNumber := int64(8934775)
|
||||
config := drip_drip.DripDripConfig
|
||||
config.StartingBlockNumber = blockNumber
|
||||
config.EndingBlockNumber = blockNumber
|
||||
|
||||
rpcClient, ethClient, err := getClients(ipc)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
blockChain, err := getBlockChain(rpcClient, ethClient)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
db := test_config.NewTestDB(blockChain.Node())
|
||||
test_config.CleanTestDB(db)
|
||||
|
||||
err = persistHeader(db, blockNumber, blockChain)
|
||||
header, err := persistHeader(db, blockNumber, blockChain)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
initializer := factories.LogNoteTransformer{
|
||||
Config: config,
|
||||
Converter: &drip_drip.DripDripConverter{},
|
||||
Repository: &drip_drip.DripDripRepository{},
|
||||
Fetcher: &shared.Fetcher{},
|
||||
}
|
||||
transformer := initializer.NewLogNoteTransformer(db, blockChain)
|
||||
transformer := initializer.NewLogNoteTransformer(db)
|
||||
|
||||
err = transformer.Execute()
|
||||
fetcher := shared.NewFetcher(blockChain)
|
||||
logs, err := fetcher.FetchLogs(
|
||||
shared.HexStringsToAddresses(config.ContractAddresses),
|
||||
[]common.Hash{common.HexToHash(config.Topic)},
|
||||
header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
err = transformer.Execute(logs, header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
var dbResults []drip_drip.DripDripModel
|
||||
|
||||
@ -15,8 +15,11 @@
|
||||
package integration_tests
|
||||
|
||||
import (
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
|
||||
|
||||
@ -25,31 +28,44 @@ import (
|
||||
)
|
||||
|
||||
var _ = Describe("Drip File Vow LogNoteTransformer", func() {
|
||||
var (
|
||||
db *postgres.DB
|
||||
blockChain core.BlockChain
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
rpcClient, ethClient, err := getClients(ipc)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
blockChain, err = getBlockChain(rpcClient, ethClient)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
db = test_config.NewTestDB(blockChain.Node())
|
||||
test_config.CleanTestDB(db)
|
||||
})
|
||||
|
||||
It("transforms DripFileVow log events", func() {
|
||||
blockNumber := int64(8762197)
|
||||
config := vow.DripFileVowConfig
|
||||
config.StartingBlockNumber = blockNumber
|
||||
config.EndingBlockNumber = blockNumber
|
||||
|
||||
rpcClient, ethClient, err := getClients(ipc)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
blockChain, err := getBlockChain(rpcClient, ethClient)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
db := test_config.NewTestDB(blockChain.Node())
|
||||
test_config.CleanTestDB(db)
|
||||
|
||||
err = persistHeader(db, blockNumber, blockChain)
|
||||
header, err := persistHeader(db, blockNumber, blockChain)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
initializer := factories.LogNoteTransformer{
|
||||
Config: config,
|
||||
Fetcher: &shared.Fetcher{},
|
||||
Converter: &vow.DripFileVowConverter{},
|
||||
Repository: &vow.DripFileVowRepository{},
|
||||
}
|
||||
transformer := initializer.NewLogNoteTransformer(db, blockChain)
|
||||
err = transformer.Execute()
|
||||
transformer := initializer.NewLogNoteTransformer(db)
|
||||
|
||||
fetcher := shared.NewFetcher(blockChain)
|
||||
logs, err := fetcher.FetchLogs(
|
||||
shared.HexStringsToAddresses(config.ContractAddresses),
|
||||
[]common.Hash{common.HexToHash(config.Topic)},
|
||||
header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
err = transformer.Execute(logs, header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
var dbResult []vow.DripFileVowModel
|
||||
|
||||
@ -15,6 +15,10 @@
|
||||
package integration_tests
|
||||
|
||||
import (
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/test_config"
|
||||
"time"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
@ -23,35 +27,46 @@ import (
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/flap_kick"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
|
||||
"github.com/vulcanize/vulcanizedb/test_config"
|
||||
)
|
||||
|
||||
var _ = Describe("FlapKick Transformer", func() {
|
||||
var (
|
||||
db *postgres.DB
|
||||
blockChain core.BlockChain
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
rpcClient, ethClient, err := getClients(ipc)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
blockChain, err = getBlockChain(rpcClient, ethClient)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
db = test_config.NewTestDB(blockChain.Node())
|
||||
test_config.CleanTestDB(db)
|
||||
})
|
||||
|
||||
It("fetches and transforms a FlapKick event from Kovan chain", func() {
|
||||
blockNumber := int64(9002933)
|
||||
config := flap_kick.FlapKickConfig
|
||||
config.StartingBlockNumber = blockNumber
|
||||
config.EndingBlockNumber = blockNumber
|
||||
|
||||
rpcClient, ethClient, err := getClients(ipc)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
blockChain, err := getBlockChain(rpcClient, ethClient)
|
||||
header, err := persistHeader(db, blockNumber, blockChain)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
db := test_config.NewTestDB(blockChain.Node())
|
||||
test_config.CleanTestDB(db)
|
||||
|
||||
err = persistHeader(db, blockNumber, blockChain)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
initializer := factories.Transformer{
|
||||
transformer := factories.Transformer{
|
||||
Config: config,
|
||||
Converter: &flap_kick.FlapKickConverter{},
|
||||
Repository: &flap_kick.FlapKickRepository{},
|
||||
Fetcher: &shared.Fetcher{},
|
||||
}
|
||||
transformer := initializer.NewTransformer(db, blockChain)
|
||||
err = transformer.Execute()
|
||||
}.NewTransformer(db)
|
||||
|
||||
fetcher := shared.NewFetcher(blockChain)
|
||||
logs, err := fetcher.FetchLogs(
|
||||
shared.HexStringsToAddresses(config.ContractAddresses),
|
||||
[]common.Hash{common.HexToHash(config.Topic)},
|
||||
header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
err = transformer.Execute(logs, header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
var dbResult []flap_kick.FlapKickModel
|
||||
|
||||
@ -65,21 +65,26 @@ var _ = Describe("FlipKick Transformer", func() {
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
blockChain, err := getBlockChain(rpcClient, ethClient)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
db := test_config.NewTestDB(blockChain.Node())
|
||||
test_config.CleanTestDB(db)
|
||||
|
||||
err = persistHeader(db, blockNumber, blockChain)
|
||||
header, err := persistHeader(db, blockNumber, blockChain)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
initializer := factories.Transformer{
|
||||
transformer := factories.Transformer{
|
||||
Config: config,
|
||||
Converter: &flip_kick.FlipKickConverter{},
|
||||
Repository: &flip_kick.FlipKickRepository{},
|
||||
Fetcher: &shared.Fetcher{},
|
||||
}
|
||||
transformer := initializer.NewTransformer(db, blockChain)
|
||||
err = transformer.Execute()
|
||||
}.NewTransformer(db)
|
||||
|
||||
fetcher := shared.NewFetcher(blockChain)
|
||||
logs, err := fetcher.FetchLogs(
|
||||
shared.HexStringsToAddresses(config.ContractAddresses),
|
||||
[]common.Hash{common.HexToHash(config.Topic)},
|
||||
header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
err = transformer.Execute(logs, header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
var dbResult []flip_kick.FlipKickModel
|
||||
|
||||
@ -35,8 +35,13 @@ import (
|
||||
|
||||
var _ = Describe("FlopKick Transformer", func() {
|
||||
var (
|
||||
db *postgres.DB
|
||||
blockChain core.BlockChain
|
||||
db *postgres.DB
|
||||
blockChain core.BlockChain
|
||||
config shared.TransformerConfig
|
||||
initializer factories.Transformer
|
||||
fetcher shared.LogFetcher
|
||||
addresses []common.Address
|
||||
topics []common.Hash
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
@ -46,25 +51,33 @@ var _ = Describe("FlopKick Transformer", func() {
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
db = test_config.NewTestDB(blockChain.Node())
|
||||
test_config.CleanTestDB(db)
|
||||
|
||||
config = flop_kick.Config
|
||||
|
||||
initializer = factories.Transformer{
|
||||
Config: config,
|
||||
Converter: &flop_kick.FlopKickConverter{},
|
||||
Repository: &flop_kick.FlopKickRepository{},
|
||||
}
|
||||
|
||||
fetcher = shared.NewFetcher(blockChain)
|
||||
addresses = shared.HexStringsToAddresses(config.ContractAddresses)
|
||||
topics = []common.Hash{common.HexToHash(config.Topic)}
|
||||
})
|
||||
|
||||
It("fetches and transforms a FlopKick event from Kovan chain", func() {
|
||||
blockNumber := int64(8672119)
|
||||
config := flop_kick.Config
|
||||
config.StartingBlockNumber = blockNumber
|
||||
config.EndingBlockNumber = blockNumber
|
||||
initializer.Config.StartingBlockNumber = blockNumber
|
||||
initializer.Config.EndingBlockNumber = blockNumber
|
||||
|
||||
err := persistHeader(db, blockNumber, blockChain)
|
||||
header, err := persistHeader(db, blockNumber, blockChain)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
initializer := factories.Transformer{
|
||||
Config: config,
|
||||
Converter: &flop_kick.FlopKickConverter{},
|
||||
Repository: &flop_kick.FlopKickRepository{},
|
||||
Fetcher: &shared.Fetcher{},
|
||||
}
|
||||
transformer := initializer.NewTransformer(db, blockChain)
|
||||
err = transformer.Execute()
|
||||
logs, err := fetcher.FetchLogs(addresses, topics, header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
transformer := initializer.NewTransformer(db)
|
||||
err = transformer.Execute(logs, header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
var dbResult []flop_kick.Model
|
||||
@ -82,21 +95,17 @@ var _ = Describe("FlopKick Transformer", func() {
|
||||
|
||||
It("fetches and transforms another FlopKick event from Kovan chain", func() {
|
||||
blockNumber := int64(8955611)
|
||||
config := flop_kick.Config
|
||||
config.StartingBlockNumber = blockNumber
|
||||
config.EndingBlockNumber = blockNumber
|
||||
initializer.Config.StartingBlockNumber = blockNumber
|
||||
initializer.Config.EndingBlockNumber = blockNumber
|
||||
|
||||
err := persistHeader(db, blockNumber, blockChain)
|
||||
header, err := persistHeader(db, blockNumber, blockChain)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
initializer := factories.Transformer{
|
||||
Config: config,
|
||||
Converter: &flop_kick.FlopKickConverter{},
|
||||
Repository: &flop_kick.FlopKickRepository{},
|
||||
Fetcher: &shared.Fetcher{},
|
||||
}
|
||||
transformer := initializer.NewTransformer(db, blockChain)
|
||||
err = transformer.Execute()
|
||||
logs, err := fetcher.FetchLogs(addresses, topics, header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
transformer := initializer.NewTransformer(db)
|
||||
err = transformer.Execute(logs, header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
var dbResult []flop_kick.Model
|
||||
|
||||
@ -33,8 +33,11 @@ import (
|
||||
|
||||
var _ = Describe("Frob Transformer", func() {
|
||||
var (
|
||||
db *postgres.DB
|
||||
blockChain core.BlockChain
|
||||
db *postgres.DB
|
||||
blockChain core.BlockChain
|
||||
fetcher *shared.Fetcher
|
||||
config shared.TransformerConfig
|
||||
initializer factories.Transformer
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
@ -44,25 +47,32 @@ var _ = Describe("Frob Transformer", func() {
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
db = test_config.NewTestDB(blockChain.Node())
|
||||
test_config.CleanTestDB(db)
|
||||
|
||||
fetcher = shared.NewFetcher(blockChain)
|
||||
config = frob.FrobConfig
|
||||
initializer = factories.Transformer{
|
||||
Config: config,
|
||||
Converter: &frob.FrobConverter{},
|
||||
Repository: &frob.FrobRepository{},
|
||||
}
|
||||
})
|
||||
|
||||
It("fetches and transforms a Frob event from Kovan chain", func() {
|
||||
blockNumber := int64(8935258)
|
||||
config := frob.FrobConfig
|
||||
config.StartingBlockNumber = blockNumber
|
||||
config.EndingBlockNumber = blockNumber
|
||||
initializer.Config.StartingBlockNumber = blockNumber
|
||||
initializer.Config.EndingBlockNumber = blockNumber
|
||||
|
||||
err := persistHeader(db, blockNumber, blockChain)
|
||||
header, err := persistHeader(db, blockNumber, blockChain)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
initializer := factories.Transformer{
|
||||
Config: config,
|
||||
Converter: &frob.FrobConverter{},
|
||||
Repository: &frob.FrobRepository{},
|
||||
Fetcher: &shared.Fetcher{},
|
||||
}
|
||||
transformer := initializer.NewTransformer(db, blockChain)
|
||||
err = transformer.Execute()
|
||||
logs, err := fetcher.FetchLogs(
|
||||
shared.HexStringsToAddresses(config.ContractAddresses),
|
||||
[]common.Hash{common.HexToHash(config.Topic)},
|
||||
header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
transformer := initializer.NewTransformer(db)
|
||||
err = transformer.Execute(logs, header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
var dbResult []frob.FrobModel
|
||||
|
||||
@ -42,12 +42,14 @@ func getBlockChain(rpcClient client.RpcClient, ethClient *ethclient.Client) (cor
|
||||
return blockChain, nil
|
||||
}
|
||||
|
||||
func persistHeader(db *postgres.DB, blockNumber int64, blockChain core.BlockChain) error {
|
||||
// Persist the header for a given block to postgres. Returns the header if successful.
|
||||
func persistHeader(db *postgres.DB, blockNumber int64, blockChain core.BlockChain) (core.Header, error) {
|
||||
header, err := blockChain.GetHeaderByNumber(blockNumber)
|
||||
if err != nil {
|
||||
return err
|
||||
return core.Header{}, err
|
||||
}
|
||||
headerRepository := repositories.NewHeaderRepository(db)
|
||||
_, err = headerRepository.CreateOrUpdateHeader(header)
|
||||
return err
|
||||
id, err := headerRepository.CreateOrUpdateHeader(header)
|
||||
header.Id = id
|
||||
return header, err
|
||||
}
|
||||
|
||||
@ -15,8 +15,11 @@
|
||||
package integration_tests
|
||||
|
||||
import (
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
|
||||
|
||||
@ -25,31 +28,44 @@ import (
|
||||
)
|
||||
|
||||
var _ = Describe("PitFileDebtCeiling LogNoteTransformer", func() {
|
||||
var (
|
||||
db *postgres.DB
|
||||
blockChain core.BlockChain
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
rpcClient, ethClient, err := getClients(ipc)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
blockChain, err = getBlockChain(rpcClient, ethClient)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
db = test_config.NewTestDB(blockChain.Node())
|
||||
test_config.CleanTestDB(db)
|
||||
})
|
||||
|
||||
It("fetches and transforms a PitFileDebtCeiling event from Kovan chain", func() {
|
||||
blockNumber := int64(8535578)
|
||||
config := debt_ceiling.DebtCeilingFileConfig
|
||||
config.StartingBlockNumber = blockNumber
|
||||
config.EndingBlockNumber = blockNumber
|
||||
|
||||
rpcClient, ethClient, err := getClients(ipc)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
blockChain, err := getBlockChain(rpcClient, ethClient)
|
||||
header, err := persistHeader(db, blockNumber, blockChain)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
db := test_config.NewTestDB(blockChain.Node())
|
||||
test_config.CleanTestDB(db)
|
||||
|
||||
err = persistHeader(db, blockNumber, blockChain)
|
||||
fetcher := shared.NewFetcher(blockChain)
|
||||
logs, err := fetcher.FetchLogs(
|
||||
shared.HexStringsToAddresses(config.ContractAddresses),
|
||||
[]common.Hash{common.HexToHash(config.Topic)},
|
||||
header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
initializer := factories.LogNoteTransformer{
|
||||
Config: config,
|
||||
Fetcher: &shared.Fetcher{},
|
||||
Converter: &debt_ceiling.PitFileDebtCeilingConverter{},
|
||||
Repository: &debt_ceiling.PitFileDebtCeilingRepository{},
|
||||
}
|
||||
transformer := initializer.NewLogNoteTransformer(db, blockChain)
|
||||
err = transformer.Execute()
|
||||
transformer := initializer.NewLogNoteTransformer(db)
|
||||
|
||||
err = transformer.Execute(logs, header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
var dbResult []debt_ceiling.PitFileDebtCeilingModel
|
||||
|
||||
@ -15,8 +15,11 @@
|
||||
package integration_tests
|
||||
|
||||
import (
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/ilk"
|
||||
@ -25,31 +28,47 @@ import (
|
||||
)
|
||||
|
||||
var _ = Describe("PitFileIlk LogNoteTransformer", func() {
|
||||
It("fetches and transforms a Pit.file ilk 'spot' event from Kovan", func() {
|
||||
blockNumber := int64(9103223)
|
||||
config := ilk.IlkFileConfig
|
||||
config.StartingBlockNumber = blockNumber
|
||||
config.EndingBlockNumber = blockNumber
|
||||
var (
|
||||
db *postgres.DB
|
||||
blockChain core.BlockChain
|
||||
initializer factories.LogNoteTransformer
|
||||
addresses []common.Address
|
||||
topics []common.Hash
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
rpcClient, ethClient, err := getClients(ipc)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
blockChain, err := getBlockChain(rpcClient, ethClient)
|
||||
blockChain, err = getBlockChain(rpcClient, ethClient)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
db := test_config.NewTestDB(blockChain.Node())
|
||||
db = test_config.NewTestDB(blockChain.Node())
|
||||
test_config.CleanTestDB(db)
|
||||
config := ilk.IlkFileConfig
|
||||
|
||||
err = persistHeader(db, blockNumber, blockChain)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
addresses = shared.HexStringsToAddresses(config.ContractAddresses)
|
||||
topics = []common.Hash{common.HexToHash(config.Topic)}
|
||||
|
||||
initializer := factories.LogNoteTransformer{
|
||||
initializer = factories.LogNoteTransformer{
|
||||
Config: config,
|
||||
Fetcher: &shared.Fetcher{},
|
||||
Converter: &ilk.PitFileIlkConverter{},
|
||||
Repository: &ilk.PitFileIlkRepository{},
|
||||
}
|
||||
transformer := initializer.NewLogNoteTransformer(db, blockChain)
|
||||
err = transformer.Execute()
|
||||
})
|
||||
|
||||
It("fetches and transforms a Pit.file ilk 'spot' event from Kovan", func() {
|
||||
blockNumber := int64(9103223)
|
||||
initializer.Config.StartingBlockNumber = blockNumber
|
||||
initializer.Config.EndingBlockNumber = blockNumber
|
||||
|
||||
header, err := persistHeader(db, blockNumber, blockChain)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
fetcher := shared.NewFetcher(blockChain)
|
||||
logs, err := fetcher.FetchLogs(addresses, topics, header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
transformer := initializer.NewLogNoteTransformer(db)
|
||||
err = transformer.Execute(logs, header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
var dbResult []ilk.PitFileIlkModel
|
||||
@ -64,29 +83,18 @@ var _ = Describe("PitFileIlk LogNoteTransformer", func() {
|
||||
|
||||
It("fetches and transforms a Pit.file ilk 'line' event from Kovan", func() {
|
||||
blockNumber := int64(8762253)
|
||||
config := ilk.IlkFileConfig
|
||||
config.StartingBlockNumber = blockNumber
|
||||
config.EndingBlockNumber = blockNumber
|
||||
initializer.Config.StartingBlockNumber = blockNumber
|
||||
initializer.Config.EndingBlockNumber = blockNumber
|
||||
|
||||
rpcClient, ethClient, err := getClients(ipc)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
blockChain, err := getBlockChain(rpcClient, ethClient)
|
||||
header, err := persistHeader(db, blockNumber, blockChain)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
db := test_config.NewTestDB(blockChain.Node())
|
||||
test_config.CleanTestDB(db)
|
||||
|
||||
err = persistHeader(db, blockNumber, blockChain)
|
||||
fetcher := shared.NewFetcher(blockChain)
|
||||
logs, err := fetcher.FetchLogs(addresses, topics, header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
initializer := factories.LogNoteTransformer{
|
||||
Config: config,
|
||||
Fetcher: &shared.Fetcher{},
|
||||
Converter: &ilk.PitFileIlkConverter{},
|
||||
Repository: &ilk.PitFileIlkRepository{},
|
||||
}
|
||||
transformer := initializer.NewLogNoteTransformer(db, blockChain)
|
||||
err = transformer.Execute()
|
||||
transformer := initializer.NewLogNoteTransformer(db)
|
||||
err = transformer.Execute(logs, header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
var dbResult []ilk.PitFileIlkModel
|
||||
|
||||
@ -15,6 +15,7 @@
|
||||
package integration_tests
|
||||
|
||||
import (
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
@ -29,8 +30,12 @@ import (
|
||||
|
||||
var _ = Describe("Price feeds transformer", func() {
|
||||
var (
|
||||
db *postgres.DB
|
||||
blockChain core.BlockChain
|
||||
db *postgres.DB
|
||||
blockChain core.BlockChain
|
||||
config shared.TransformerConfig
|
||||
fetcher *shared.Fetcher
|
||||
initializer factories.LogNoteTransformer
|
||||
topics []common.Hash
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
@ -40,86 +45,94 @@ var _ = Describe("Price feeds transformer", func() {
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
db = test_config.NewTestDB(blockChain.Node())
|
||||
test_config.CleanTestDB(db)
|
||||
|
||||
config = price_feeds.PriceFeedConfig
|
||||
topics = []common.Hash{common.HexToHash(config.Topic)}
|
||||
|
||||
fetcher = shared.NewFetcher(blockChain)
|
||||
|
||||
initializer = factories.LogNoteTransformer{
|
||||
Config: config,
|
||||
Converter: &price_feeds.PriceFeedConverter{},
|
||||
Repository: &price_feeds.PriceFeedRepository{},
|
||||
}
|
||||
})
|
||||
|
||||
It("persists a ETH/USD price feed event", func() {
|
||||
blockNumber := int64(8763054)
|
||||
err := persistHeader(db, blockNumber, blockChain)
|
||||
header, err := persistHeader(db, blockNumber, blockChain)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
config := price_feeds.PriceFeedConfig
|
||||
config.ContractAddresses = []string{constants.PipContractAddress}
|
||||
config.StartingBlockNumber = blockNumber
|
||||
config.EndingBlockNumber = blockNumber
|
||||
addresses := []string{constants.PipContractAddress}
|
||||
initializer.Config.ContractAddresses = addresses
|
||||
initializer.Config.StartingBlockNumber = blockNumber
|
||||
initializer.Config.EndingBlockNumber = blockNumber
|
||||
|
||||
transformerInitializer := factories.LogNoteTransformer{
|
||||
Config: config,
|
||||
Converter: &price_feeds.PriceFeedConverter{},
|
||||
Repository: &price_feeds.PriceFeedRepository{},
|
||||
Fetcher: &shared.Fetcher{},
|
||||
}
|
||||
transformer := transformerInitializer.NewLogNoteTransformer(db, blockChain)
|
||||
logs, err := fetcher.FetchLogs(
|
||||
shared.HexStringsToAddresses(addresses),
|
||||
topics,
|
||||
header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
err = transformer.Execute()
|
||||
transformer := initializer.NewLogNoteTransformer(db)
|
||||
err = transformer.Execute(logs, header)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
var model price_feeds.PriceFeedModel
|
||||
err = db.Get(&model, `SELECT block_number, medianizer_address, usd_value, tx_idx, raw_log FROM maker.price_feeds WHERE block_number = $1`, config.StartingBlockNumber)
|
||||
err = db.Get(&model, `SELECT block_number, medianizer_address, usd_value, tx_idx, raw_log FROM maker.price_feeds WHERE block_number = $1`, initializer.Config.StartingBlockNumber)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(model.UsdValue).To(Equal("207.314891143000011198"))
|
||||
Expect(model.MedianizerAddress).To(Equal(config.ContractAddresses[0]))
|
||||
Expect(model.MedianizerAddress).To(Equal(addresses[0]))
|
||||
})
|
||||
|
||||
It("persists a MKR/USD price feed event", func() {
|
||||
blockNumber := int64(8763059)
|
||||
err := persistHeader(db, blockNumber, blockChain)
|
||||
header, err := persistHeader(db, blockNumber, blockChain)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
config := price_feeds.PriceFeedConfig
|
||||
config.ContractAddresses = []string{constants.PepContractAddress}
|
||||
config.StartingBlockNumber = blockNumber
|
||||
config.EndingBlockNumber = blockNumber
|
||||
addresses := []string{constants.PepContractAddress}
|
||||
initializer.Config.ContractAddresses = addresses
|
||||
initializer.Config.StartingBlockNumber = blockNumber
|
||||
initializer.Config.EndingBlockNumber = blockNumber
|
||||
|
||||
transformerInitializer := factories.LogNoteTransformer{
|
||||
Config: config,
|
||||
Converter: &price_feeds.PriceFeedConverter{},
|
||||
Repository: &price_feeds.PriceFeedRepository{},
|
||||
Fetcher: &shared.Fetcher{},
|
||||
}
|
||||
transformer := transformerInitializer.NewLogNoteTransformer(db, blockChain)
|
||||
logs, err := fetcher.FetchLogs(
|
||||
shared.HexStringsToAddresses(addresses),
|
||||
topics,
|
||||
header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
err = transformer.Execute()
|
||||
transformer := initializer.NewLogNoteTransformer(db)
|
||||
err = transformer.Execute(logs, header)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
var model price_feeds.PriceFeedModel
|
||||
err = db.Get(&model, `SELECT block_number, medianizer_address, usd_value, tx_idx, raw_log FROM maker.price_feeds WHERE block_number = $1`, config.StartingBlockNumber)
|
||||
err = db.Get(&model, `SELECT block_number, medianizer_address, usd_value, tx_idx, raw_log FROM maker.price_feeds WHERE block_number = $1`, initializer.Config.StartingBlockNumber)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(model.UsdValue).To(Equal("391.803979212000001553"))
|
||||
Expect(model.MedianizerAddress).To(Equal(config.ContractAddresses[0]))
|
||||
Expect(model.MedianizerAddress).To(Equal(addresses[0]))
|
||||
})
|
||||
|
||||
It("persists a REP/USD price feed event", func() {
|
||||
blockNumber := int64(8763062)
|
||||
err := persistHeader(db, blockNumber, blockChain)
|
||||
header, err := persistHeader(db, blockNumber, blockChain)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
config := price_feeds.PriceFeedConfig
|
||||
config.ContractAddresses = []string{constants.RepContractAddress}
|
||||
config.StartingBlockNumber = blockNumber
|
||||
config.EndingBlockNumber = blockNumber
|
||||
addresses := []string{constants.RepContractAddress}
|
||||
initializer.Config.ContractAddresses = addresses
|
||||
initializer.Config.StartingBlockNumber = blockNumber
|
||||
initializer.Config.EndingBlockNumber = blockNumber
|
||||
|
||||
transformerInitializer := factories.LogNoteTransformer{
|
||||
Config: config,
|
||||
Converter: &price_feeds.PriceFeedConverter{},
|
||||
Repository: &price_feeds.PriceFeedRepository{},
|
||||
Fetcher: &shared.Fetcher{},
|
||||
}
|
||||
transformer := transformerInitializer.NewLogNoteTransformer(db, blockChain)
|
||||
logs, err := fetcher.FetchLogs(
|
||||
shared.HexStringsToAddresses(addresses),
|
||||
topics,
|
||||
header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
err = transformer.Execute()
|
||||
transformer := initializer.NewLogNoteTransformer(db)
|
||||
err = transformer.Execute(logs, header)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
var model price_feeds.PriceFeedModel
|
||||
err = db.Get(&model, `SELECT block_number, medianizer_address, usd_value, tx_idx, raw_log FROM maker.price_feeds WHERE block_number = $1`, config.StartingBlockNumber)
|
||||
err = db.Get(&model, `SELECT block_number, medianizer_address, usd_value, tx_idx, raw_log FROM maker.price_feeds WHERE block_number = $1`, initializer.Config.StartingBlockNumber)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(model.UsdValue).To(Equal("12.816928482699999847"))
|
||||
Expect(model.MedianizerAddress).To(Equal(config.ContractAddresses[0]))
|
||||
Expect(model.MedianizerAddress).To(Equal(addresses[0]))
|
||||
})
|
||||
})
|
||||
|
||||
@ -15,6 +15,7 @@
|
||||
package integration_tests
|
||||
|
||||
import (
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
@ -28,8 +29,13 @@ import (
|
||||
|
||||
var _ = Describe("Tend LogNoteTransformer", func() {
|
||||
var (
|
||||
db *postgres.DB
|
||||
blockChain core.BlockChain
|
||||
db *postgres.DB
|
||||
blockChain core.BlockChain
|
||||
config shared.TransformerConfig
|
||||
fetcher *shared.Fetcher
|
||||
initializer factories.LogNoteTransformer
|
||||
addresses []common.Address
|
||||
topics []common.Hash
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
@ -39,25 +45,33 @@ var _ = Describe("Tend LogNoteTransformer", func() {
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
db = test_config.NewTestDB(blockChain.Node())
|
||||
test_config.CleanTestDB(db)
|
||||
|
||||
config = tend.TendConfig
|
||||
|
||||
fetcher = shared.NewFetcher(blockChain)
|
||||
addresses = shared.HexStringsToAddresses(config.ContractAddresses)
|
||||
topics = []common.Hash{common.HexToHash(config.Topic)}
|
||||
|
||||
initializer = factories.LogNoteTransformer{
|
||||
Config: config,
|
||||
Converter: &tend.TendConverter{},
|
||||
Repository: &tend.TendRepository{},
|
||||
}
|
||||
})
|
||||
|
||||
It("fetches and transforms a Flip Tend event from Kovan chain", func() {
|
||||
blockNumber := int64(8935601)
|
||||
config := tend.TendConfig
|
||||
config.StartingBlockNumber = blockNumber
|
||||
config.EndingBlockNumber = blockNumber
|
||||
initializer.Config.StartingBlockNumber = blockNumber
|
||||
initializer.Config.EndingBlockNumber = blockNumber
|
||||
|
||||
err := persistHeader(db, blockNumber, blockChain)
|
||||
header, err := persistHeader(db, blockNumber, blockChain)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
initializer := factories.LogNoteTransformer{
|
||||
Config: config,
|
||||
Fetcher: &shared.Fetcher{},
|
||||
Converter: &tend.TendConverter{},
|
||||
Repository: &tend.TendRepository{},
|
||||
}
|
||||
transformer := initializer.NewLogNoteTransformer(db, blockChain)
|
||||
err = transformer.Execute()
|
||||
logs, err := fetcher.FetchLogs(addresses, topics, header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
transformer := initializer.NewLogNoteTransformer(db)
|
||||
err = transformer.Execute(logs, header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
var dbResult []tend.TendModel
|
||||
@ -80,21 +94,17 @@ var _ = Describe("Tend LogNoteTransformer", func() {
|
||||
|
||||
It("fetches and transforms a subsequent Flip Tend event from Kovan chain for the same auction", func() {
|
||||
blockNumber := int64(8935731)
|
||||
config := tend.TendConfig
|
||||
config.StartingBlockNumber = blockNumber
|
||||
config.EndingBlockNumber = blockNumber
|
||||
initializer.Config.StartingBlockNumber = blockNumber
|
||||
initializer.Config.EndingBlockNumber = blockNumber
|
||||
|
||||
err := persistHeader(db, blockNumber, blockChain)
|
||||
header, err := persistHeader(db, blockNumber, blockChain)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
initializer := factories.LogNoteTransformer{
|
||||
Config: config,
|
||||
Fetcher: &shared.Fetcher{},
|
||||
Converter: &tend.TendConverter{},
|
||||
Repository: &tend.TendRepository{},
|
||||
}
|
||||
transformer := initializer.NewLogNoteTransformer(db, blockChain)
|
||||
err = transformer.Execute()
|
||||
logs, err := fetcher.FetchLogs(addresses, topics, header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
transformer := initializer.NewLogNoteTransformer(db)
|
||||
err = transformer.Execute(logs, header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
var dbResult []tend.TendModel
|
||||
@ -117,21 +127,17 @@ var _ = Describe("Tend LogNoteTransformer", func() {
|
||||
|
||||
It("fetches and transforms a Flap Tend event from the Kovan chain", func() {
|
||||
blockNumber := int64(9003177)
|
||||
config := tend.TendConfig
|
||||
config.StartingBlockNumber = blockNumber
|
||||
config.EndingBlockNumber = blockNumber
|
||||
initializer.Config.StartingBlockNumber = blockNumber
|
||||
initializer.Config.EndingBlockNumber = blockNumber
|
||||
|
||||
err := persistHeader(db, blockNumber, blockChain)
|
||||
header, err := persistHeader(db, blockNumber, blockChain)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
initializer := factories.LogNoteTransformer{
|
||||
Config: config,
|
||||
Fetcher: &shared.Fetcher{},
|
||||
Converter: &tend.TendConverter{},
|
||||
Repository: &tend.TendRepository{},
|
||||
}
|
||||
transformer := initializer.NewLogNoteTransformer(db, blockChain)
|
||||
err = transformer.Execute()
|
||||
logs, err := fetcher.FetchLogs(addresses, topics, header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
transformer := initializer.NewLogNoteTransformer(db)
|
||||
err = transformer.Execute(logs, header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
var dbResult []tend.TendModel
|
||||
|
||||
@ -15,6 +15,7 @@
|
||||
package integration_tests
|
||||
|
||||
import (
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
@ -39,17 +40,24 @@ var _ = Describe("VatFlux LogNoteTransformer", func() {
|
||||
db := test_config.NewTestDB(blockChain.Node())
|
||||
test_config.CleanTestDB(db)
|
||||
|
||||
err = persistHeader(db, blockNumber, blockChain)
|
||||
header, err := persistHeader(db, blockNumber, blockChain)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
fetcher := shared.NewFetcher(blockChain)
|
||||
logs, err := fetcher.FetchLogs(
|
||||
shared.HexStringsToAddresses(config.ContractAddresses),
|
||||
[]common.Hash{common.HexToHash(config.Topic)},
|
||||
header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
initializer := factories.LogNoteTransformer{
|
||||
Config: config,
|
||||
Fetcher: &shared.Fetcher{},
|
||||
Converter: &vat_flux.VatFluxConverter{},
|
||||
Repository: &vat_flux.VatFluxRepository{},
|
||||
}
|
||||
transformer := initializer.NewLogNoteTransformer(db, blockChain)
|
||||
err = transformer.Execute()
|
||||
transformer := initializer.NewLogNoteTransformer(db)
|
||||
|
||||
err = transformer.Execute(logs, header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
var dbResult []vat_flux.VatFluxModel
|
||||
|
||||
@ -18,39 +18,53 @@ import (
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/test_config"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/vat_fold"
|
||||
"github.com/vulcanize/vulcanizedb/test_config"
|
||||
)
|
||||
|
||||
var _ = Describe("VatFold Transformer", func() {
|
||||
var (
|
||||
db *postgres.DB
|
||||
blockChain core.BlockChain
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
rpcClient, ethClient, err := getClients(ipc)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
blockChain, err = getBlockChain(rpcClient, ethClient)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
db = test_config.NewTestDB(blockChain.Node())
|
||||
test_config.CleanTestDB(db)
|
||||
})
|
||||
|
||||
It("transforms VatFold log events", func() {
|
||||
blockNumber := int64(9367233)
|
||||
config := vat_fold.VatFoldConfig
|
||||
config.StartingBlockNumber = blockNumber
|
||||
config.EndingBlockNumber = blockNumber
|
||||
|
||||
rpcClient, ethClient, err := getClients(ipc)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
blockChain, err := getBlockChain(rpcClient, ethClient)
|
||||
header, err := persistHeader(db, blockNumber, blockChain)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
db := test_config.NewTestDB(blockChain.Node())
|
||||
test_config.CleanTestDB(db)
|
||||
|
||||
err = persistHeader(db, blockNumber, blockChain)
|
||||
fetcher := shared.NewFetcher(blockChain)
|
||||
logs, err := fetcher.FetchLogs(
|
||||
shared.HexStringsToAddresses(config.ContractAddresses),
|
||||
[]common.Hash{common.HexToHash(config.Topic)},
|
||||
header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
initializer := factories.LogNoteTransformer{
|
||||
transformer := factories.LogNoteTransformer{
|
||||
Config: config,
|
||||
Fetcher: &shared.Fetcher{},
|
||||
Converter: &vat_fold.VatFoldConverter{},
|
||||
Repository: &vat_fold.VatFoldRepository{},
|
||||
}
|
||||
transformer := initializer.NewLogNoteTransformer(db, blockChain)
|
||||
err = transformer.Execute()
|
||||
}.NewLogNoteTransformer(db)
|
||||
|
||||
err = transformer.Execute(logs, header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
var dbResults []vat_fold.VatFoldModel
|
||||
|
||||
@ -15,6 +15,7 @@
|
||||
package integration_tests
|
||||
|
||||
import (
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"math/big"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
@ -41,17 +42,23 @@ var _ = Describe("Vat Grab Transformer", func() {
|
||||
db := test_config.NewTestDB(blockChain.Node())
|
||||
test_config.CleanTestDB(db)
|
||||
|
||||
err = persistHeader(db, blockNumber, blockChain)
|
||||
header, err := persistHeader(db, blockNumber, blockChain)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
fetcher := shared.NewFetcher(blockChain)
|
||||
logs, err := fetcher.FetchLogs(
|
||||
shared.HexStringsToAddresses(config.ContractAddresses),
|
||||
[]common.Hash{common.HexToHash(config.Topic)},
|
||||
header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
transformer := factories.LogNoteTransformer{
|
||||
Config: config,
|
||||
Converter: &vat_grab.VatGrabConverter{},
|
||||
Repository: &vat_grab.VatGrabRepository{},
|
||||
Fetcher: &shared.Fetcher{},
|
||||
}.NewLogNoteTransformer(db, blockChain)
|
||||
}.NewLogNoteTransformer(db)
|
||||
|
||||
err = transformer.Execute()
|
||||
err = transformer.Execute(logs, header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
var dbResult []vat_grab.VatGrabModel
|
||||
|
||||
@ -40,17 +40,23 @@ var _ = Describe("VatHeal Transformer", func() {
|
||||
db := test_config.NewTestDB(blockChain.Node())
|
||||
test_config.CleanTestDB(db)
|
||||
|
||||
err = persistHeader(db, blockNumber, blockChain)
|
||||
header, err := persistHeader(db, blockNumber, blockChain)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
fetcher := shared.NewFetcher(blockChain)
|
||||
logs, err := fetcher.FetchLogs(
|
||||
shared.HexStringsToAddresses(config.ContractAddresses),
|
||||
[]common.Hash{common.HexToHash(config.Topic)},
|
||||
header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
transformer := factories.LogNoteTransformer{
|
||||
Config: config,
|
||||
Converter: &vat_heal.VatHealConverter{},
|
||||
Repository: &vat_heal.VatHealRepository{},
|
||||
Fetcher: &shared.Fetcher{},
|
||||
}.NewLogNoteTransformer(db, blockChain)
|
||||
}.NewLogNoteTransformer(db)
|
||||
|
||||
err = transformer.Execute()
|
||||
err = transformer.Execute(logs, header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
var dbResults []vat_heal.VatHealModel
|
||||
|
||||
@ -15,6 +15,7 @@
|
||||
package integration_tests
|
||||
|
||||
import (
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
|
||||
@ -39,17 +40,23 @@ var _ = Describe("VatInit LogNoteTransformer", func() {
|
||||
db := test_config.NewTestDB(blockChain.Node())
|
||||
test_config.CleanTestDB(db)
|
||||
|
||||
err = persistHeader(db, blockNumber, blockChain)
|
||||
header, err := persistHeader(db, blockNumber, blockChain)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
initializer := factories.LogNoteTransformer{
|
||||
fetcher := shared.NewFetcher(blockChain)
|
||||
logs, err := fetcher.FetchLogs(
|
||||
shared.HexStringsToAddresses(config.ContractAddresses),
|
||||
[]common.Hash{common.HexToHash(config.Topic)},
|
||||
header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
transformer := factories.LogNoteTransformer{
|
||||
Config: config,
|
||||
Fetcher: &shared.Fetcher{},
|
||||
Converter: &vat_init.VatInitConverter{},
|
||||
Repository: &vat_init.VatInitRepository{},
|
||||
}
|
||||
transformer := initializer.NewLogNoteTransformer(db, blockChain)
|
||||
err = transformer.Execute()
|
||||
}.NewLogNoteTransformer(db)
|
||||
|
||||
err = transformer.Execute(logs, header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
var dbResults []vat_init.VatInitModel
|
||||
|
||||
@ -40,17 +40,23 @@ var _ = Describe("VatMove LogNoteTransformer", func() {
|
||||
db := test_config.NewTestDB(blockChain.Node())
|
||||
test_config.CleanTestDB(db)
|
||||
|
||||
err = persistHeader(db, blockNumber, blockChain)
|
||||
header, err := persistHeader(db, blockNumber, blockChain)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
initializer := factories.LogNoteTransformer{
|
||||
fetcher := shared.NewFetcher(blockChain)
|
||||
logs, err := fetcher.FetchLogs(
|
||||
shared.HexStringsToAddresses(config.ContractAddresses),
|
||||
[]common.Hash{common.HexToHash(config.Topic)},
|
||||
header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
transformer := factories.LogNoteTransformer{
|
||||
Config: config,
|
||||
Fetcher: &shared.Fetcher{},
|
||||
Converter: &vat_move.VatMoveConverter{},
|
||||
Repository: &vat_move.VatMoveRepository{},
|
||||
}
|
||||
transformer := initializer.NewLogNoteTransformer(db, blockChain)
|
||||
err = transformer.Execute()
|
||||
}.NewLogNoteTransformer(db)
|
||||
|
||||
err = transformer.Execute(logs, header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
var dbResults []vat_move.VatMoveModel
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
package integration_tests
|
||||
|
||||
import (
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
@ -18,31 +19,38 @@ var _ = Describe("Vat slip transformer", func() {
|
||||
blockChain core.BlockChain
|
||||
)
|
||||
|
||||
It("persists vat slip event", func() {
|
||||
BeforeEach(func() {
|
||||
rpcClient, ethClient, err := getClients(ipc)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
blockChain, err = getBlockChain(rpcClient, ethClient)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
db = test_config.NewTestDB(blockChain.Node())
|
||||
test_config.CleanTestDB(db)
|
||||
})
|
||||
|
||||
It("persists vat slip event", func() {
|
||||
blockNumber := int64(8953655)
|
||||
config := vat_slip.VatSlipConfig
|
||||
config.StartingBlockNumber = blockNumber
|
||||
config.EndingBlockNumber = blockNumber
|
||||
|
||||
err = persistHeader(db, blockNumber, blockChain)
|
||||
header, err := persistHeader(db, blockNumber, blockChain)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
initializer := factories.LogNoteTransformer{
|
||||
fetcher := shared.NewFetcher(blockChain)
|
||||
logs, err := fetcher.FetchLogs(
|
||||
shared.HexStringsToAddresses(config.ContractAddresses),
|
||||
[]common.Hash{common.HexToHash(config.Topic)},
|
||||
header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
transformer := factories.LogNoteTransformer{
|
||||
Config: config,
|
||||
Fetcher: &shared.Fetcher{},
|
||||
Converter: &vat_slip.VatSlipConverter{},
|
||||
Repository: &vat_slip.VatSlipRepository{},
|
||||
}
|
||||
transformer := initializer.NewLogNoteTransformer(db, blockChain)
|
||||
}.NewLogNoteTransformer(db)
|
||||
|
||||
err = transformer.Execute()
|
||||
err = transformer.Execute(logs, header)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
var headerID int64
|
||||
|
||||
@ -15,6 +15,7 @@
|
||||
package integration_tests
|
||||
|
||||
import (
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"math/big"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
@ -41,17 +42,23 @@ var _ = Describe("VatTune LogNoteTransformer", func() {
|
||||
db := test_config.NewTestDB(blockChain.Node())
|
||||
test_config.CleanTestDB(db)
|
||||
|
||||
err = persistHeader(db, blockNumber, blockChain)
|
||||
header, err := persistHeader(db, blockNumber, blockChain)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
initializer := factories.LogNoteTransformer{
|
||||
fetcher := shared.NewFetcher(blockChain)
|
||||
logs, err := fetcher.FetchLogs(
|
||||
shared.HexStringsToAddresses(config.ContractAddresses),
|
||||
[]common.Hash{common.HexToHash(config.Topic)},
|
||||
header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
transformer := factories.LogNoteTransformer{
|
||||
Config: config,
|
||||
Fetcher: &shared.Fetcher{},
|
||||
Converter: &vat_tune.VatTuneConverter{},
|
||||
Repository: &vat_tune.VatTuneRepository{},
|
||||
}
|
||||
transformer := initializer.NewLogNoteTransformer(db, blockChain)
|
||||
err = transformer.Execute()
|
||||
}.NewLogNoteTransformer(db)
|
||||
|
||||
err = transformer.Execute(logs, header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
var dbResult []vat_tune.VatTuneModel
|
||||
|
||||
@ -15,8 +15,11 @@
|
||||
package integration_tests
|
||||
|
||||
import (
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
|
||||
@ -25,32 +28,43 @@ import (
|
||||
)
|
||||
|
||||
var _ = Describe("VowFlog LogNoteTransformer", func() {
|
||||
var (
|
||||
db *postgres.DB
|
||||
blockChain core.BlockChain
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
rpcClient, ethClient, err := getClients(ipc)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
blockChain, err = getBlockChain(rpcClient, ethClient)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
db = test_config.NewTestDB(blockChain.Node())
|
||||
test_config.CleanTestDB(db)
|
||||
})
|
||||
|
||||
It("transforms VowFlog log events", func() {
|
||||
blockNumber := int64(8946819)
|
||||
config := vow_flog.VowFlogConfig
|
||||
config.StartingBlockNumber = blockNumber
|
||||
config.EndingBlockNumber = blockNumber
|
||||
|
||||
rpcClient, ethClient, err := getClients(ipc)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
blockChain, err := getBlockChain(rpcClient, ethClient)
|
||||
header, err := persistHeader(db, blockNumber, blockChain)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
db := test_config.NewTestDB(blockChain.Node())
|
||||
test_config.CleanTestDB(db)
|
||||
|
||||
err = persistHeader(db, blockNumber, blockChain)
|
||||
fetcher := shared.NewFetcher(blockChain)
|
||||
logs, err := fetcher.FetchLogs(
|
||||
shared.HexStringsToAddresses(config.ContractAddresses),
|
||||
[]common.Hash{common.HexToHash(config.Topic)},
|
||||
header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(1).To(Equal(1))
|
||||
|
||||
initializer := factories.LogNoteTransformer{
|
||||
transformer := factories.LogNoteTransformer{
|
||||
Config: config,
|
||||
Fetcher: &shared.Fetcher{},
|
||||
Converter: &vow_flog.VowFlogConverter{},
|
||||
Repository: &vow_flog.VowFlogRepository{},
|
||||
}
|
||||
transformer := initializer.NewLogNoteTransformer(db, blockChain)
|
||||
err = transformer.Execute()
|
||||
}.NewLogNoteTransformer(db)
|
||||
|
||||
err = transformer.Execute(logs, header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
var dbResult []vow_flog.VowFlogModel
|
||||
|
||||
@ -16,7 +16,6 @@ package debt_ceiling
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
|
||||
@ -64,10 +63,6 @@ func (repository PitFileDebtCeilingRepository) MarkHeaderChecked(headerID int64)
|
||||
return shared.MarkHeaderChecked(headerID, repository.db, constants.PitFileDebtCeilingChecked)
|
||||
}
|
||||
|
||||
func (repository PitFileDebtCeilingRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
|
||||
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.PitFileDebtCeilingChecked)
|
||||
}
|
||||
|
||||
func (repository *PitFileDebtCeilingRepository) SetDB(db *postgres.DB) {
|
||||
repository.db = db
|
||||
}
|
||||
|
||||
@ -81,13 +81,4 @@ var _ = Describe("Pit file debt ceiling repository", func() {
|
||||
|
||||
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
|
||||
})
|
||||
|
||||
Describe("MissingHeaders", func() {
|
||||
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
|
||||
Repository: &pitFileDebtCeilingRepository,
|
||||
RepositoryTwo: &debt_ceiling.PitFileDebtCeilingRepository{},
|
||||
}
|
||||
|
||||
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
|
||||
})
|
||||
})
|
||||
|
||||
@ -16,7 +16,6 @@ package ilk
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
|
||||
@ -62,10 +61,6 @@ func (repository PitFileIlkRepository) MarkHeaderChecked(headerID int64) error {
|
||||
return shared.MarkHeaderChecked(headerID, repository.db, constants.PitFileIlkChecked)
|
||||
}
|
||||
|
||||
func (repository PitFileIlkRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
|
||||
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.PitFileIlkChecked)
|
||||
}
|
||||
|
||||
func (repository *PitFileIlkRepository) SetDB(db *postgres.DB) {
|
||||
repository.db = db
|
||||
}
|
||||
|
||||
@ -82,13 +82,4 @@ var _ = Describe("Pit file ilk repository", func() {
|
||||
|
||||
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
|
||||
})
|
||||
|
||||
Describe("MissingHeaders", func() {
|
||||
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
|
||||
Repository: &pitFileIlkRepository,
|
||||
RepositoryTwo: &ilk.PitFileIlkRepository{},
|
||||
}
|
||||
|
||||
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
|
||||
})
|
||||
})
|
||||
|
||||
@ -16,7 +16,6 @@ package price_feeds
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
|
||||
@ -58,10 +57,6 @@ func (repository PriceFeedRepository) MarkHeaderChecked(headerID int64) error {
|
||||
return shared.MarkHeaderChecked(headerID, repository.db, constants.PriceFeedsChecked)
|
||||
}
|
||||
|
||||
func (repository PriceFeedRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
|
||||
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.PriceFeedsChecked)
|
||||
}
|
||||
|
||||
func (repository *PriceFeedRepository) SetDB(db *postgres.DB) {
|
||||
repository.db = db
|
||||
}
|
||||
|
||||
@ -82,13 +82,4 @@ var _ = Describe("Price feeds repository", func() {
|
||||
|
||||
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
|
||||
})
|
||||
|
||||
Describe("MissingHeaders", func() {
|
||||
inputs := shared_behaviors.MissingHeadersBehaviorInputs{
|
||||
Repository: &priceFeedRepository,
|
||||
RepositoryTwo: &price_feeds.PriceFeedRepository{},
|
||||
}
|
||||
|
||||
shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs)
|
||||
})
|
||||
})
|
||||
|
||||
65
pkg/transformers/shared/log_chunker.go
Normal file
65
pkg/transformers/shared/log_chunker.go
Normal file
@ -0,0 +1,65 @@
|
||||
// Copyright 2018 Vulcanize
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package shared
|
||||
|
||||
import (
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
)
|
||||
|
||||
type Chunker interface {
|
||||
AddConfigs(transformerConfigs []TransformerConfig)
|
||||
ChunkLogs(logs []types.Log) map[string][]types.Log
|
||||
}
|
||||
|
||||
type LogChunker struct {
|
||||
AddressToNames map[string][]string
|
||||
NameToTopic0 map[string]common.Hash
|
||||
}
|
||||
|
||||
// Returns a new log chunker with initialised maps.
|
||||
// Needs to have configs added with `AddConfigs` to consider logs for the respective transformer.
|
||||
func NewLogChunker() *LogChunker {
|
||||
return &LogChunker{
|
||||
AddressToNames: map[string][]string{},
|
||||
NameToTopic0: map[string]common.Hash{},
|
||||
}
|
||||
}
|
||||
|
||||
// Configures the chunker by adding more addreses and topics to consider.
|
||||
func (chunker *LogChunker) AddConfigs(transformerConfigs []TransformerConfig) {
|
||||
for _, config := range transformerConfigs {
|
||||
for _, address := range config.ContractAddresses {
|
||||
chunker.AddressToNames[address] = append(chunker.AddressToNames[address], config.TransformerName)
|
||||
chunker.NameToTopic0[config.TransformerName] = common.HexToHash(config.Topic)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Goes through an array of logs, associating relevant logs (matching addresses and topic) with transformers
|
||||
func (chunker *LogChunker) ChunkLogs(logs []types.Log) map[string][]types.Log {
|
||||
chunks := map[string][]types.Log{}
|
||||
for _, log := range logs {
|
||||
// Topic0 is not unique to each transformer, also need to consider the contract address
|
||||
relevantTransformers := chunker.AddressToNames[log.Address.String()]
|
||||
|
||||
for _, transformer := range relevantTransformers {
|
||||
if chunker.NameToTopic0[transformer] == log.Topics[0] {
|
||||
chunks[transformer] = append(chunks[transformer], log)
|
||||
}
|
||||
}
|
||||
}
|
||||
return chunks
|
||||
}
|
||||
137
pkg/transformers/shared/log_chunker_test.go
Normal file
137
pkg/transformers/shared/log_chunker_test.go
Normal file
@ -0,0 +1,137 @@
|
||||
// Copyright 2018 Vulcanize
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package shared_test
|
||||
|
||||
import (
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
|
||||
)
|
||||
|
||||
var _ = Describe("Log chunker", func() {
|
||||
var (
|
||||
configs []shared.TransformerConfig
|
||||
chunker *shared.LogChunker
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
configA := shared.TransformerConfig{
|
||||
TransformerName: "TransformerA",
|
||||
ContractAddresses: []string{"0x00000000000000000000000000000000000000A1", "0x00000000000000000000000000000000000000A2"},
|
||||
Topic: "0xA",
|
||||
}
|
||||
configB := shared.TransformerConfig{
|
||||
TransformerName: "TransformerB",
|
||||
ContractAddresses: []string{"0x00000000000000000000000000000000000000B1"},
|
||||
Topic: "0xB",
|
||||
}
|
||||
|
||||
configC := shared.TransformerConfig{
|
||||
TransformerName: "TransformerC",
|
||||
ContractAddresses: []string{"0x00000000000000000000000000000000000000A2"},
|
||||
Topic: "0xC",
|
||||
}
|
||||
|
||||
configs = []shared.TransformerConfig{configA, configB, configC}
|
||||
chunker = shared.NewLogChunker()
|
||||
chunker.AddConfigs(configs)
|
||||
})
|
||||
|
||||
Describe("initialisation", func() {
|
||||
It("creates lookup maps correctly", func() {
|
||||
Expect(chunker.AddressToNames).To(Equal(map[string][]string{
|
||||
"0x00000000000000000000000000000000000000A1": []string{"TransformerA"},
|
||||
"0x00000000000000000000000000000000000000A2": []string{"TransformerA", "TransformerC"},
|
||||
"0x00000000000000000000000000000000000000B1": []string{"TransformerB"},
|
||||
}))
|
||||
|
||||
Expect(chunker.NameToTopic0).To(Equal(map[string]common.Hash{
|
||||
"TransformerA": common.HexToHash("0xA"),
|
||||
"TransformerB": common.HexToHash("0xB"),
|
||||
"TransformerC": common.HexToHash("0xC"),
|
||||
}))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("AddConfigs", func() {
|
||||
It("can add more configs later", func() {
|
||||
configD := shared.TransformerConfig{
|
||||
TransformerName: "TransformerD",
|
||||
ContractAddresses: []string{"0x000000000000000000000000000000000000000D"},
|
||||
Topic: "0xD",
|
||||
}
|
||||
chunker.AddConfigs([]shared.TransformerConfig{configD})
|
||||
|
||||
Expect(chunker.AddressToNames).To(ContainElement([]string{"TransformerD"}))
|
||||
Expect(chunker.NameToTopic0).To(ContainElement(common.HexToHash("0xD")))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("ChunkLogs", func() {
|
||||
It("only associates logs with relevant topic0 and address to transformers", func() {
|
||||
logs := []types.Log{log1, log2, log3, log4, log5}
|
||||
chunks := chunker.ChunkLogs(logs)
|
||||
|
||||
Expect(chunks["TransformerA"]).To(And(ContainElement(log1), ContainElement(log4)))
|
||||
Expect(chunks["TransformerB"]).To(BeEmpty())
|
||||
Expect(chunks["TransformerC"]).To(ContainElement(log5))
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
var (
|
||||
// Match TransformerA
|
||||
log1 = types.Log{
|
||||
Address: common.HexToAddress("0xA1"),
|
||||
Topics: []common.Hash{
|
||||
common.HexToHash("0xA"),
|
||||
common.HexToHash("0xLogTopic1"),
|
||||
},
|
||||
}
|
||||
// Match TransformerA address, but not topic0
|
||||
log2 = types.Log{
|
||||
Address: common.HexToAddress("0xA1"),
|
||||
Topics: []common.Hash{
|
||||
common.HexToHash("0xB"),
|
||||
common.HexToHash("0xLogTopic2"),
|
||||
},
|
||||
}
|
||||
// Match TransformerA topic, but TransformerB address
|
||||
log3 = types.Log{
|
||||
Address: common.HexToAddress("0xB1"),
|
||||
Topics: []common.Hash{
|
||||
common.HexToHash("0xA"),
|
||||
common.HexToHash("0xLogTopic3"),
|
||||
},
|
||||
}
|
||||
// Match TransformerA, with the other address
|
||||
log4 = types.Log{
|
||||
Address: common.HexToAddress("0xA2"),
|
||||
Topics: []common.Hash{
|
||||
common.HexToHash("0xA"),
|
||||
common.HexToHash("0xLogTopic4"),
|
||||
},
|
||||
}
|
||||
// Match TransformerC, which shares address with TransformerA
|
||||
log5 = types.Log{
|
||||
Address: common.HexToAddress("0xA2"),
|
||||
Topics: []common.Hash{
|
||||
common.HexToHash("0xC"),
|
||||
common.HexToHash("0xLogTopic5"),
|
||||
},
|
||||
}
|
||||
)
|
||||
@ -23,45 +23,34 @@ import (
|
||||
)
|
||||
|
||||
type LogFetcher interface {
|
||||
FetchLogs(contractAddresses []string, topics [][]common.Hash, header core.Header) ([]types.Log, error)
|
||||
}
|
||||
|
||||
type SettableLogFetcher interface {
|
||||
LogFetcher
|
||||
SetBC(bc core.BlockChain)
|
||||
FetchLogs(contractAddresses []common.Address, topics []common.Hash, missingHeader core.Header) ([]types.Log, error)
|
||||
}
|
||||
|
||||
type Fetcher struct {
|
||||
blockChain core.BlockChain
|
||||
}
|
||||
|
||||
func (fetcher *Fetcher) SetBC(bc core.BlockChain) {
|
||||
fetcher.blockChain = bc
|
||||
}
|
||||
|
||||
func NewFetcher(blockchain core.BlockChain) Fetcher {
|
||||
return Fetcher{
|
||||
func NewFetcher(blockchain core.BlockChain) *Fetcher {
|
||||
return &Fetcher{
|
||||
blockChain: blockchain,
|
||||
}
|
||||
}
|
||||
|
||||
func (fetcher Fetcher) FetchLogs(contractAddresses []string, topics [][]common.Hash, header core.Header) ([]types.Log, error) {
|
||||
addresses := hexStringsToAddresses(contractAddresses)
|
||||
// Checks all topic0s, on all addresses, fetching matching logs for the given header
|
||||
func (fetcher Fetcher) FetchLogs(addresses []common.Address, topic0s []common.Hash, header core.Header) ([]types.Log, error) {
|
||||
blockHash := common.HexToHash(header.Hash)
|
||||
query := ethereum.FilterQuery{
|
||||
BlockHash: &blockHash,
|
||||
Addresses: addresses,
|
||||
Topics: topics,
|
||||
}
|
||||
return fetcher.blockChain.GetEthLogsWithCustomQuery(query)
|
||||
}
|
||||
|
||||
func hexStringsToAddresses(hexStrings []string) []common.Address {
|
||||
var addresses []common.Address
|
||||
for _, hexString := range hexStrings {
|
||||
address := common.HexToAddress(hexString)
|
||||
addresses = append(addresses, address)
|
||||
// Search for _any_ of the topics in topic0 position; see docs on `FilterQuery`
|
||||
Topics: [][]common.Hash{topic0s},
|
||||
}
|
||||
|
||||
return addresses
|
||||
logs, err := fetcher.blockChain.GetEthLogsWithCustomQuery(query)
|
||||
if err != nil {
|
||||
// TODO review aggregate fetching error handling
|
||||
return []types.Log{}, err
|
||||
}
|
||||
|
||||
return logs, nil
|
||||
}
|
||||
|
||||
@ -32,8 +32,12 @@ var _ = Describe("Fetcher", func() {
|
||||
fetcher := shared.NewFetcher(blockChain)
|
||||
header := fakes.FakeHeader
|
||||
|
||||
addresses := []string{"0xfakeAddress", "0xanotherFakeAddress"}
|
||||
topicZeros := [][]common.Hash{{common.BytesToHash([]byte{1, 2, 3, 4, 5})}}
|
||||
addresses := []common.Address{
|
||||
common.HexToAddress("0xfakeAddress"),
|
||||
common.HexToAddress("0xanotherFakeAddress"),
|
||||
}
|
||||
|
||||
topicZeros := []common.Hash{common.BytesToHash([]byte{1, 2, 3, 4, 5})}
|
||||
|
||||
_, err := fetcher.FetchLogs(addresses, topicZeros, header)
|
||||
|
||||
@ -45,7 +49,7 @@ var _ = Describe("Fetcher", func() {
|
||||
expectedQuery := ethereum.FilterQuery{
|
||||
BlockHash: &blockHash,
|
||||
Addresses: []common.Address{address1, address2},
|
||||
Topics: topicZeros,
|
||||
Topics: [][]common.Hash{topicZeros},
|
||||
}
|
||||
blockChain.AssertGetEthLogsWithCustomQueryCalledWith(expectedQuery)
|
||||
})
|
||||
@ -55,7 +59,7 @@ var _ = Describe("Fetcher", func() {
|
||||
blockChain.SetGetEthLogsWithCustomQueryErr(fakes.FakeError)
|
||||
fetcher := shared.NewFetcher(blockChain)
|
||||
|
||||
_, err := fetcher.FetchLogs([]string{}, [][]common.Hash{}, core.Header{})
|
||||
_, err := fetcher.FetchLogs([]common.Address{}, []common.Hash{}, core.Header{})
|
||||
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err).To(MatchError(fakes.FakeError))
|
||||
|
||||
@ -1,7 +1,10 @@
|
||||
package shared
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"database/sql"
|
||||
"database/sql/driver"
|
||||
"fmt"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
)
|
||||
@ -22,7 +25,8 @@ func MarkHeaderCheckedInTransaction(headerID int64, tx *sql.Tx, checkedHeadersCo
|
||||
return err
|
||||
}
|
||||
|
||||
func MissingHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.DB, checkedHeadersColumn string) ([]core.Header, error) {
|
||||
// Treats a header as missing if it's not in the headers table, or not checked for some log type
|
||||
func MissingHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.DB, notCheckedSQL string) ([]core.Header, error) {
|
||||
var result []core.Header
|
||||
var query string
|
||||
var err error
|
||||
@ -30,14 +34,14 @@ func MissingHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.D
|
||||
if endingBlockNumber == -1 {
|
||||
query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
|
||||
LEFT JOIN checked_headers on headers.id = header_id
|
||||
WHERE (header_id ISNULL OR ` + checkedHeadersColumn + ` IS FALSE)
|
||||
WHERE (header_id ISNULL OR ` + notCheckedSQL + `)
|
||||
AND headers.block_number >= $1
|
||||
AND headers.eth_node_fingerprint = $2`
|
||||
err = db.Select(&result, query, startingBlockNumber, db.Node.ID)
|
||||
} else {
|
||||
query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
|
||||
LEFT JOIN checked_headers on headers.id = header_id
|
||||
WHERE (header_id ISNULL OR ` + checkedHeadersColumn + ` IS FALSE)
|
||||
WHERE (header_id ISNULL OR ` + notCheckedSQL + `)
|
||||
AND headers.block_number >= $1
|
||||
AND headers.block_number <= $2
|
||||
AND headers.eth_node_fingerprint = $3`
|
||||
@ -46,3 +50,53 @@ func MissingHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.D
|
||||
|
||||
return result, err
|
||||
}
|
||||
|
||||
func GetCheckedColumnNames(db *postgres.DB) ([]string, error) {
|
||||
// Query returns `[]driver.Value`, nullable polymorphic interface
|
||||
var queryResult []driver.Value
|
||||
columnNamesQuery :=
|
||||
`SELECT column_name FROM information_schema.columns
|
||||
WHERE table_schema = 'public'
|
||||
AND table_name = 'checked_headers'
|
||||
AND column_name ~ '_checked';`
|
||||
|
||||
err := db.Select(&queryResult, columnNamesQuery)
|
||||
if err != nil {
|
||||
return []string{}, err
|
||||
}
|
||||
|
||||
// Transform column names from `driver.Value` to strings
|
||||
var columnNames []string
|
||||
for _, result := range queryResult {
|
||||
if columnName, ok := result.(string); ok {
|
||||
columnNames = append(columnNames, columnName)
|
||||
} else {
|
||||
return []string{}, fmt.Errorf("incorrect value for checked_headers column name")
|
||||
}
|
||||
}
|
||||
|
||||
return columnNames, nil
|
||||
}
|
||||
|
||||
// Builds a SQL string that checks if any column value is FALSE, given the column names.
|
||||
// Defaults to FALSE when no columns are provided.
|
||||
// Ex: ["columnA", "columnB"] => "NOT (columnA AND columnB)"
|
||||
// [] => "FALSE"
|
||||
func CreateNotCheckedSQL(boolColumns []string) string {
|
||||
var result bytes.Buffer
|
||||
|
||||
if len(boolColumns) == 0 {
|
||||
return "FALSE"
|
||||
}
|
||||
|
||||
result.WriteString("NOT (")
|
||||
// Loop excluding last column name
|
||||
for _, column := range boolColumns[:len(boolColumns)-1] {
|
||||
result.WriteString(fmt.Sprintf("%v AND ", column))
|
||||
}
|
||||
|
||||
// No trailing "OR" for the last column name
|
||||
result.WriteString(fmt.Sprintf("%v)", boolColumns[len(boolColumns)-1]))
|
||||
|
||||
return result.String()
|
||||
}
|
||||
|
||||
170
pkg/transformers/shared/repository_utility_test.go
Normal file
170
pkg/transformers/shared/repository_utility_test.go
Normal file
@ -0,0 +1,170 @@
|
||||
// Copyright 2018 Vulcanize
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package shared_test
|
||||
|
||||
import (
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/fakes"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
|
||||
"github.com/vulcanize/vulcanizedb/test_config"
|
||||
"math/rand"
|
||||
)
|
||||
|
||||
var _ = Describe("Repository utilities", func() {
|
||||
Describe("MissingHeaders", func() {
|
||||
var (
|
||||
db *postgres.DB
|
||||
headerRepository datastore.HeaderRepository
|
||||
startingBlockNumber int64
|
||||
endingBlockNumber int64
|
||||
eventSpecificBlockNumber int64
|
||||
blockNumbers []int64
|
||||
headerIDs []int64
|
||||
notCheckedSQL string
|
||||
err error
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
db = test_config.NewTestDB(test_config.NewTestNode())
|
||||
test_config.CleanTestDB(db)
|
||||
headerRepository = repositories.NewHeaderRepository(db)
|
||||
|
||||
columnNames, err := shared.GetCheckedColumnNames(db)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
notCheckedSQL = shared.CreateNotCheckedSQL(columnNames)
|
||||
|
||||
startingBlockNumber = rand.Int63()
|
||||
eventSpecificBlockNumber = startingBlockNumber + 1
|
||||
endingBlockNumber = startingBlockNumber + 2
|
||||
outOfRangeBlockNumber := endingBlockNumber + 1
|
||||
|
||||
blockNumbers = []int64{startingBlockNumber, eventSpecificBlockNumber, endingBlockNumber, outOfRangeBlockNumber}
|
||||
|
||||
headerIDs = []int64{}
|
||||
for _, n := range blockNumbers {
|
||||
headerID, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(n))
|
||||
headerIDs = append(headerIDs, headerID)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
}
|
||||
})
|
||||
|
||||
It("only treats headers as checked if the event specific logs have been checked", func() {
|
||||
_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
headers, err := shared.MissingHeaders(startingBlockNumber, endingBlockNumber, db, notCheckedSQL)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(len(headers)).To(Equal(3))
|
||||
Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber)))
|
||||
Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber)))
|
||||
Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber)))
|
||||
})
|
||||
|
||||
It("only returns headers associated with the current node", func() {
|
||||
dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
|
||||
headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
|
||||
for _, n := range blockNumbers {
|
||||
_, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n + 10))
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
}
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
nodeOneMissingHeaders, err := shared.MissingHeaders(startingBlockNumber, endingBlockNumber, db, notCheckedSQL)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(len(nodeOneMissingHeaders)).To(Equal(3))
|
||||
Expect(nodeOneMissingHeaders[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(eventSpecificBlockNumber), Equal(endingBlockNumber)))
|
||||
Expect(nodeOneMissingHeaders[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(eventSpecificBlockNumber), Equal(endingBlockNumber)))
|
||||
Expect(nodeOneMissingHeaders[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(startingBlockNumber), Equal(eventSpecificBlockNumber), Equal(endingBlockNumber)))
|
||||
|
||||
nodeTwoMissingHeaders, err := shared.MissingHeaders(startingBlockNumber, endingBlockNumber+10, dbTwo, notCheckedSQL)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(len(nodeTwoMissingHeaders)).To(Equal(3))
|
||||
Expect(nodeTwoMissingHeaders[0].BlockNumber).To(Or(Equal(startingBlockNumber+10), Equal(eventSpecificBlockNumber+10), Equal(endingBlockNumber+10)))
|
||||
Expect(nodeTwoMissingHeaders[1].BlockNumber).To(Or(Equal(startingBlockNumber+10), Equal(eventSpecificBlockNumber+10), Equal(endingBlockNumber+10)))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("GetCheckedColumnNames", func() {
|
||||
It("gets the column names from checked_headers", func() {
|
||||
db := test_config.NewTestDB(test_config.NewTestNode())
|
||||
test_config.CleanTestDB(db)
|
||||
expectedColumnNames := getExpectedColumnNames()
|
||||
actualColumnNames, err := shared.GetCheckedColumnNames(db)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(actualColumnNames).To(Equal(expectedColumnNames))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("CreateNotCheckedSQL", func() {
|
||||
It("generates a correct SQL string for one column", func() {
|
||||
columns := []string{"columnA"}
|
||||
expected := "NOT (columnA)"
|
||||
actual := shared.CreateNotCheckedSQL(columns)
|
||||
Expect(actual).To(Equal(expected))
|
||||
})
|
||||
|
||||
It("generates a correct SQL string for several columns", func() {
|
||||
columns := []string{"columnA", "columnB"}
|
||||
expected := "NOT (columnA AND columnB)"
|
||||
actual := shared.CreateNotCheckedSQL(columns)
|
||||
Expect(actual).To(Equal(expected))
|
||||
})
|
||||
|
||||
It("defaults to FALSE when there are no columns", func() {
|
||||
expected := "FALSE"
|
||||
actual := shared.CreateNotCheckedSQL([]string{})
|
||||
Expect(actual).To(Equal(expected))
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
func getExpectedColumnNames() []string {
|
||||
return []string{
|
||||
"price_feeds_checked",
|
||||
"flip_kick_checked",
|
||||
"frob_checked",
|
||||
"tend_checked",
|
||||
"bite_checked",
|
||||
"dent_checked",
|
||||
"pit_file_debt_ceiling_checked",
|
||||
"pit_file_ilk_checked",
|
||||
"vat_init_checked",
|
||||
"drip_file_ilk_checked",
|
||||
"drip_file_repo_checked",
|
||||
"drip_file_vow_checked",
|
||||
"deal_checked",
|
||||
"drip_drip_checked",
|
||||
"cat_file_chop_lump_checked",
|
||||
"cat_file_flip_checked",
|
||||
"cat_file_pit_vow_checked",
|
||||
"flop_kick_checked",
|
||||
"vat_move_checked",
|
||||
"vat_fold_checked",
|
||||
"vat_heal_checked",
|
||||
"vat_toll_checked",
|
||||
"vat_tune_checked",
|
||||
"vat_grab_checked",
|
||||
"vat_flux_checked",
|
||||
"vat_slip_checked",
|
||||
"vow_flog_checked",
|
||||
"flap_kick_checked",
|
||||
}
|
||||
}
|
||||
@ -16,16 +16,18 @@ package shared
|
||||
|
||||
import (
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
)
|
||||
|
||||
type Transformer interface {
|
||||
Execute() error
|
||||
Execute(logs []types.Log, header core.Header) error
|
||||
GetConfig() TransformerConfig
|
||||
}
|
||||
|
||||
type TransformerInitializer func(db *postgres.DB, blockChain core.BlockChain) Transformer
|
||||
type TransformerInitializer func(db *postgres.DB) Transformer
|
||||
|
||||
type TransformerConfig struct {
|
||||
TransformerName string
|
||||
@ -45,3 +47,10 @@ func HexToString(byteString string) string {
|
||||
value := common.HexToHash(byteString)
|
||||
return value.Big().String()
|
||||
}
|
||||
|
||||
func HexStringsToAddresses(strings []string) (addresses []common.Address) {
|
||||
for _, hexString := range strings {
|
||||
addresses = append(addresses, common.HexToAddress(hexString))
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
@ -16,7 +16,6 @@ package tend
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
|
||||
@ -68,10 +67,6 @@ func (repository TendRepository) MarkHeaderChecked(headerId int64) error {
|
||||
return shared.MarkHeaderChecked(headerId, repository.db, constants.TendChecked)
|
||||
}
|
||||
|
||||
func (repository TendRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
|
||||
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.TendChecked)
|
||||
}
|
||||
|
||||
func (repository *TendRepository) SetDB(db *postgres.DB) {
|
||||
repository.db = db
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user