cerc-io/ipld-eth-server
18
0
Fork 1
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases 37 Wiki Activity

Combine price feed transformers

Browse Source 
- fetches logs from all three price feeds in one query
- assumes eth/usd price feed will be updated to include LogValue event
- updates transformers to run separate from header sync
This commit is contained in:
Rob Mulholand 2018-08-14 16:59:41 -05:00
parent 9231d40369
commit 634604d0b5
68 changed files with 902 additions and 1384 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
README.md
View File

@ -29,8 +29,7 @@ Vulcanize DB is a set of tools that make it easier for developers to write appli
## Configuration
- To use a local Ethereum node, copy `environments/public.toml.example` to
`environments/public.toml` and update the `ipcPath`, `levelDbPath`,
`pipContractAddress`, `pepContractAddress`, and `repContractAddress`.
`environments/public.toml` and update the `ipcPath` and `levelDbPath`.
- `ipcPath` should match the local node's IPC filepath:
- when using geth:
- The IPC file is called `geth.ipc`.
@ -52,10 +51,7 @@ Vulcanize DB is a set of tools that make it easier for developers to write appli
- Linux: `$HOME/.ethereum/geth/chaindata`
- `levelDbPath` is irrelevant (and `coldImport` is currently unavailable) if only running parity.
- `pepContractAddress`, `pipContractAddress`, and `repContractAddress` should match that medianizer
addresses for each pair on the chain you're tracking. See https://makerdao.com/feeds/
- See `environments/infura.toml` to configure commands to run against infura, if a local node is unavailable
- See `environments/infura.toml` to configure commands to run against infura, if a local node is unavailable.
- Copy `environments/local.toml.example` to `environments/local.toml` to configure commands to run against a local node such as [Ganache](https://truffleframework.com/ganache) or [ganache-cli](https://github.com/trufflesuite/ganache-clihttps://github.com/trufflesuite/ganache-cli).
## Start syncing with postgres
@ -82,22 +78,17 @@ This command is useful when you want a minimal baseline from which to track targ
1. In a separate terminal start VulcanizeDB:
- `./vulcanizedb lightSync --config <config.toml> --starting-block-number <block-number>`
## Backfill Auction event logs from light sync
Backfills auction event logs from the configured Ethereum node based on the populated block headers.
## Backfill Maker event logs from light sync
Backfills Maker event logs from the configured Ethereum node based on the populated block headers.
This includes logs related to auctions, multi-collateral dai, and price feeds.
This command requires that a light sync (see command above) has previously been run.
_Since auction contracts have not yet been deployed, this command will need to be run a local blockchain at the moment. As such, a new environment file will need to be added. See `environments/local.toml.example`._
_Since auction/mcd contracts have not yet been deployed, this command will need to be run a local blockchain at the moment. As such, a new environment file will need to be added. See `environments/local.toml.example`._
1. Start Ethereum node
1. In a separate terminal run the backfill command:
- `./vulcanizedb backfillAuctionLogs --config <config.toml>`
- `./vulcanizedb backfillMakerLogs --config <config.toml>`
## Sync in light mode with MakerDAO price feeds
Sync VulcanizeDB with the configured Ethereum node, populating block headers as well as price feeds for MKR/USD, ETH/USD, and REP/USD.
1. Start Ethereum node
1. In a separate terminal window start VulcanizeDB
- `./vulcanizedb syncPriceFeeds --config <config.toml> --starting-block-number <block-number>`
## Start full environment in docker by single command
### Geth Rinkeby

3
cmd/backfillMakerLogs.go
View File

@ -35,7 +35,8 @@ var backfillMakerLogsCmd = &cobra.Command{
Use: "backfillMakerLogs",
Short: "Backfill Maker event logs",
Long: `Backfills Maker event logs based on previously populated block Header records.
This currently includes logs related to Multi-collateral Dai (frob) and Auctions (flip-kick).
This currently includes logs related to Multi-collateral Dai (frob), Auctions (flip-kick),
and Price Feeds (ETH/USD, MKR/USD, and REP/USD - LogValue).
vulcanize backfillMakerLogs --config environments/local.toml

6
cmd/lightSync.go
View File

@ -31,7 +31,6 @@ import (
vRpc "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
"github.com/vulcanize/vulcanizedb/pkg/geth/node"
"github.com/vulcanize/vulcanizedb/pkg/history"
"github.com/vulcanize/vulcanizedb/pkg/transformers"
"github.com/vulcanize/vulcanizedb/utils"
)
@ -65,8 +64,7 @@ func init() {
}
func backFillAllHeaders(blockchain core.BlockChain, headerRepository datastore.HeaderRepository, missingBlocksPopulated chan int, startingBlockNumber int64) {
emptyTransformers := []transformers.Transformer{}
populated, err := history.PopulateMissingHeaders(blockchain, headerRepository, startingBlockNumber, emptyTransformers)
populated, err := history.PopulateMissingHeaders(blockchain, headerRepository, startingBlockNumber)
if err != nil {
log.Fatal("Error populating headers: ", err)
}
@ -81,7 +79,7 @@ func lightSync() {
db := utils.LoadPostgres(databaseConfig, blockChain.Node())
headerRepository := repositories.NewHeaderRepository(&db)
validator := history.NewHeaderValidator(blockChain, headerRepository, validationWindow, []transformers.Transformer{})
validator := history.NewHeaderValidator(blockChain, headerRepository, validationWindow)
missingBlocksPopulated := make(chan int)
go backFillAllHeaders(blockChain, headerRepository, missingBlocksPopulated, startingBlockNumber)

12
cmd/root.go
View File

@ -29,9 +29,6 @@ var (
databaseConfig config.Database
ipc string
levelDbPath string
pepContractAddress string
pipContractAddress string
repContractAddress string
startingBlockNumber int64
syncAll bool
endingBlockNumber int64
@ -52,9 +49,6 @@ func Execute() {
func database(cmd *cobra.Command, args []string) {
ipc = viper.GetString("client.ipcpath")
levelDbPath = viper.GetString("client.leveldbpath")
pepContractAddress = viper.GetString("client.pepcontractaddress")
pipContractAddress = viper.GetString("client.pipcontractaddress")
repContractAddress = viper.GetString("client.repcontractaddress")
databaseConfig = config.Database{
Name: viper.GetString("database.name"),
Hostname: viper.GetString("database.hostname"),
@ -76,9 +70,6 @@ func init() {
rootCmd.PersistentFlags().String("database-password", "", "database password")
rootCmd.PersistentFlags().String("client-ipcPath", "", "location of geth.ipc file")
rootCmd.PersistentFlags().String("client-levelDbPath", "", "location of levelDb chaindata")
rootCmd.PersistentFlags().String("client-pepContractAddress", "", "mkr/usd price feed contract address")
rootCmd.PersistentFlags().String("client-pipContractAddress", "", "eth/usd price feed contract address")
rootCmd.PersistentFlags().String("client-repContractAddress", "", "rep/usd price feed contract address")
viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name"))
viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port"))
@ -87,9 +78,6 @@ func init() {
viper.BindPFlag("database.password", rootCmd.PersistentFlags().Lookup("database-password"))
viper.BindPFlag("client.ipcPath", rootCmd.PersistentFlags().Lookup("client-ipcPath"))
viper.BindPFlag("client.levelDbPath", rootCmd.PersistentFlags().Lookup("client-levelDbPath"))
viper.BindPFlag("client.pepContractAddress", rootCmd.PersistentFlags().Lookup("client-pepContractAddress"))
viper.BindPFlag("client.pipContractAddress", rootCmd.PersistentFlags().Lookup("client-pipContractAddress"))
viper.BindPFlag("client.repContractAddress", rootCmd.PersistentFlags().Lookup("client-repContractAddress"))
}
func initConfig() {

88
cmd/syncPriceFeeds.go
View File

@ -1,88 +0,0 @@
// Copyright © 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cmd
import (
"log"
"os"
"time"
"github.com/spf13/cobra"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/history"
"github.com/vulcanize/vulcanizedb/pkg/transformers"
"github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds/pep"
"github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds/pip"
"github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds/rep"
"github.com/vulcanize/vulcanizedb/utils"
)
// syncPriceFeedsCmd represents the syncPriceFeeds command
var syncPriceFeedsCmd = &cobra.Command{
Use: "syncPriceFeeds",
Short: "Sync block headers with price feed data",
Long: `Sync Ethereum block headers and price feed data. For example:
./vulcanizedb syncPriceFeeds --config <config.toml> --starting-block-number <block-number>
Price feed data will be updated when price feed contracts log value events.`,
Run: func(cmd *cobra.Command, args []string) {
syncPriceFeeds()
},
}
func init() {
rootCmd.AddCommand(syncPriceFeedsCmd)
syncPriceFeedsCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "block number at which to start tracking price feeds")
}
func backFillPriceFeeds(blockchain core.BlockChain, headerRepository datastore.HeaderRepository, missingBlocksPopulated chan int, startingBlockNumber int64, transformers []transformers.Transformer) {
populated, err := history.PopulateMissingHeaders(blockchain, headerRepository, startingBlockNumber, transformers)
if err != nil {
log.Fatal("Error populating headers: ", err)
}
missingBlocksPopulated <- populated
}
func syncPriceFeeds() {
ticker := time.NewTicker(pollingInterval)
defer ticker.Stop()
blockChain := getBlockChain()
validateArgs(blockChain)
db := utils.LoadPostgres(databaseConfig, blockChain.Node())
transformers := []transformers.Transformer{
pep.NewPepTransformer(blockChain, &db, pepContractAddress),
pip.NewPipTransformer(blockChain, &db, pipContractAddress),
rep.NewRepTransformer(blockChain, &db, repContractAddress),
}
headerRepository := repositories.NewHeaderRepository(&db)
missingBlocksPopulated := make(chan int)
validator := history.NewHeaderValidator(blockChain, headerRepository, validationWindow, transformers)
go backFillPriceFeeds(blockChain, headerRepository, missingBlocksPopulated, startingBlockNumber, transformers)
for {
select {
case <-ticker.C:
window := validator.ValidateHeaders()
window.Log(os.Stdout)
case <-missingBlocksPopulated:
go backFillPriceFeeds(blockChain, headerRepository, missingBlocksPopulated, startingBlockNumber, transformers)
}
}
}

2
db/migrations/1532533671_create_peps_table.down.sql
View File

@ -1,2 +0,0 @@
DROP TABLE maker.peps;
DROP SCHEMA maker;

9
db/migrations/1532533671_create_peps_table.up.sql
View File

@ -1,9 +0,0 @@
CREATE TABLE maker.peps (
id SERIAL PRIMARY KEY,
block_number BIGINT NOT NULL,
header_id INTEGER NOT NULL,
usd_value NUMERIC,
CONSTRAINT headers_fk FOREIGN KEY (header_id)
REFERENCES headers (id)
ON DELETE CASCADE
);

1
db/migrations/1532623709_create_pips_table.down.sql
View File

@ -1 +0,0 @@
DROP TABLE maker.pips;

9
db/migrations/1532623709_create_pips_table.up.sql
View File

@ -1,9 +0,0 @@
CREATE TABLE maker.pips (
id SERIAL PRIMARY KEY,
block_number BIGINT NOT NULL,
header_id INTEGER NOT NULL,
usd_value NUMERIC,
CONSTRAINT headers_fk FOREIGN KEY (header_id)
REFERENCES headers (id)
ON DELETE CASCADE
);

1
db/migrations/1532635100_create_reps_table.down.sql
View File

@ -1 +0,0 @@
DROP TABLE maker.reps;

9
db/migrations/1532635100_create_reps_table.up.sql
View File

@ -1,9 +0,0 @@
CREATE TABLE maker.reps (
id SERIAL PRIMARY KEY,
block_number BIGINT NOT NULL,
header_id INTEGER NOT NULL,
usd_value NUMERIC,
CONSTRAINT headers_fk FOREIGN KEY (header_id)
REFERENCES headers (id)
ON DELETE CASCADE
);

1
db/migrations/1534275305_create_price_feeds_table.down.sql Normal file
View File

@ -0,0 +1 @@
DROP TABLE maker.price_feeds;

12
db/migrations/1534275305_create_price_feeds_table.up.sql Normal file
View File

@ -0,0 +1,12 @@
CREATE TABLE maker.price_feeds (
id SERIAL PRIMARY KEY,
block_number BIGINT NOT NULL,
header_id INTEGER NOT NULL,
medianizer_address bytea,
tx_idx INTEGER NOT NULL,
usd_value NUMERIC,
UNIQUE (header_id, medianizer_address, tx_idx),
CONSTRAINT headers_fk FOREIGN KEY (header_id)
REFERENCES headers (id)
ON DELETE CASCADE
);

136
db/schema.sql
View File

@ -120,22 +120,24 @@ ALTER SEQUENCE maker.frob_id_seq OWNED BY maker.frob.id;
--
-- Name: peps; Type: TABLE; Schema: maker; Owner: -
-- Name: price_feeds; Type: TABLE; Schema: maker; Owner: -
--
CREATE TABLE maker.peps (
CREATE TABLE maker.price_feeds (
id integer NOT NULL,
block_number bigint NOT NULL,
header_id integer NOT NULL,
medianizer_address bytea,
tx_idx integer NOT NULL,
usd_value numeric
);
--
-- Name: peps_id_seq; Type: SEQUENCE; Schema: maker; Owner: -
-- Name: price_feeds_id_seq; Type: SEQUENCE; Schema: maker; Owner: -
--
CREATE SEQUENCE maker.peps_id_seq
CREATE SEQUENCE maker.price_feeds_id_seq
AS integer
START WITH 1
INCREMENT BY 1
@ -145,74 +147,10 @@ CREATE SEQUENCE maker.peps_id_seq
--
-- Name: peps_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: -
-- Name: price_feeds_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: -
--
ALTER SEQUENCE maker.peps_id_seq OWNED BY maker.peps.id;
--
-- Name: pips; Type: TABLE; Schema: maker; Owner: -
--
CREATE TABLE maker.pips (
id integer NOT NULL,
block_number bigint NOT NULL,
header_id integer NOT NULL,
usd_value numeric
);
--
-- Name: pips_id_seq; Type: SEQUENCE; Schema: maker; Owner: -
--
CREATE SEQUENCE maker.pips_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: pips_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: -
--
ALTER SEQUENCE maker.pips_id_seq OWNED BY maker.pips.id;
--
-- Name: reps; Type: TABLE; Schema: maker; Owner: -
--
CREATE TABLE maker.reps (
id integer NOT NULL,
block_number bigint NOT NULL,
header_id integer NOT NULL,
usd_value numeric
);
--
-- Name: reps_id_seq; Type: SEQUENCE; Schema: maker; Owner: -
--
CREATE SEQUENCE maker.reps_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: reps_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: -
--
ALTER SEQUENCE maker.reps_id_seq OWNED BY maker.reps.id;
ALTER SEQUENCE maker.price_feeds_id_seq OWNED BY maker.price_feeds.id;
--
@ -602,24 +540,10 @@ ALTER TABLE ONLY maker.frob ALTER COLUMN id SET DEFAULT nextval('maker.frob_id_s
--
-- Name: peps id; Type: DEFAULT; Schema: maker; Owner: -
-- Name: price_feeds id; Type: DEFAULT; Schema: maker; Owner: -
--
ALTER TABLE ONLY maker.peps ALTER COLUMN id SET DEFAULT nextval('maker.peps_id_seq'::regclass);
--
-- Name: pips id; Type: DEFAULT; Schema: maker; Owner: -
--
ALTER TABLE ONLY maker.pips ALTER COLUMN id SET DEFAULT nextval('maker.pips_id_seq'::regclass);
--
-- Name: reps id; Type: DEFAULT; Schema: maker; Owner: -
--
ALTER TABLE ONLY maker.reps ALTER COLUMN id SET DEFAULT nextval('maker.reps_id_seq'::regclass);
ALTER TABLE ONLY maker.price_feeds ALTER COLUMN id SET DEFAULT nextval('maker.price_feeds_id_seq'::regclass);
--
@ -718,27 +642,19 @@ ALTER TABLE ONLY maker.frob
--
-- Name: peps peps_pkey; Type: CONSTRAINT; Schema: maker; Owner: -
-- Name: price_feeds price_feeds_header_id_medianizer_address_tx_idx_key; Type: CONSTRAINT; Schema: maker; Owner: -
--
ALTER TABLE ONLY maker.peps
ADD CONSTRAINT peps_pkey PRIMARY KEY (id);
ALTER TABLE ONLY maker.price_feeds
ADD CONSTRAINT price_feeds_header_id_medianizer_address_tx_idx_key UNIQUE (header_id, medianizer_address, tx_idx);
--
-- Name: pips pips_pkey; Type: CONSTRAINT; Schema: maker; Owner: -
-- Name: price_feeds price_feeds_pkey; Type: CONSTRAINT; Schema: maker; Owner: -
--
ALTER TABLE ONLY maker.pips
ADD CONSTRAINT pips_pkey PRIMARY KEY (id);
--
-- Name: reps reps_pkey; Type: CONSTRAINT; Schema: maker; Owner: -
--
ALTER TABLE ONLY maker.reps
ADD CONSTRAINT reps_pkey PRIMARY KEY (id);
ALTER TABLE ONLY maker.price_feeds
ADD CONSTRAINT price_feeds_pkey PRIMARY KEY (id);
--
@ -889,26 +805,10 @@ ALTER TABLE ONLY maker.frob
--
-- Name: peps headers_fk; Type: FK CONSTRAINT; Schema: maker; Owner: -
-- Name: price_feeds headers_fk; Type: FK CONSTRAINT; Schema: maker; Owner: -
--
ALTER TABLE ONLY maker.peps
ADD CONSTRAINT headers_fk FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE;
--
-- Name: pips headers_fk; Type: FK CONSTRAINT; Schema: maker; Owner: -
--
ALTER TABLE ONLY maker.pips
ADD CONSTRAINT headers_fk FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE;
--
-- Name: reps headers_fk; Type: FK CONSTRAINT; Schema: maker; Owner: -
--
ALTER TABLE ONLY maker.reps
ALTER TABLE ONLY maker.price_feeds
ADD CONSTRAINT headers_fk FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE;

5
environments/infura.toml
View File

@ -1,10 +1,7 @@
[database]
name = "vulcanize_public"
name = "vulcanize_private"
hostname = "localhost"
port = 5432
[client]
ipcPath = "https://mainnet.infura.io/J5Vd2fRtGsw0zZ0Ov3BL"
pepContractAddress = "0x99041F808D598B782D5a3e498681C2452A31da08"
pipContractAddress = "0x729D19f657BD0614b4985Cf1D82531c67569197B"
repContractAddress = "0xF5f94b7F9De14D43112e713835BCef2d55b76c1C"

3
environments/private.toml
View File

@ -5,6 +5,3 @@ port = 5432
[client]
ipcPath = "http://127.0.0.1:7545"
pepContractAddress = "0x99041F808D598B782D5a3e498681C2452A31da08"
pipContractAddress = "0x729D19f657BD0614b4985Cf1D82531c67569197B"
repContractAddress = "0xF5f94b7F9De14D43112e713835BCef2d55b76c1C"

3
environments/public.toml.example
View File

@ -6,6 +6,3 @@ port = 5432
[client]
ipcPath = <local node's IPC filepath>
levelDbPath = <local node's LevelDB chaindata filepath>
pepContractAddress = <pep medianizer's address - see https://makerdao.com/feeds/ >
pipContractAddress = <pip medianizer's address - see https://makerdao.com/feeds/ >
repContractAddress = <rep medianizer's address - see https://makerdao.com/feeds/ >

1
pkg/datastore/postgres/repositories/header_repository.go
View File

@ -3,6 +3,7 @@ package repositories
import (
"database/sql"
"errors"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)

44
pkg/fakes/mock_transformer.go
View File

@ -1,44 +0,0 @@
package fakes
import (
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
)
type MockTransformer struct {
passedHeader core.Header
passedHeaderID int64
executeCalled bool
executeErr error
}
func NewMockTransformer() *MockTransformer {
return &MockTransformer{
passedHeader: core.Header{},
passedHeaderID: 0,
executeCalled: false,
executeErr: nil,
}
}
func (transformer *MockTransformer) SetExecuteErr(err error) {
transformer.executeErr = err
}
func (transformer *MockTransformer) Execute(header core.Header, headerID int64) error {
transformer.executeCalled = true
transformer.passedHeader = header
transformer.passedHeaderID = headerID
return transformer.executeErr
}
func (transformer *MockTransformer) AssertExecuteCalledWith(header core.Header, headerID int64) {
Expect(transformer.executeCalled).To(BeTrue())
Expect(header).To(Equal(transformer.passedHeader))
Expect(headerID).To(Equal(transformer.passedHeaderID))
}
func (tranformer *MockTransformer) AssertExecuteNotCalled() {
Expect(tranformer.executeCalled).To(BeFalse())
}

7
pkg/history/header_validator.go
View File

@ -3,28 +3,25 @@ package history
import (
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore"
"github.com/vulcanize/vulcanizedb/pkg/transformers"
)
type HeaderValidator struct {
blockChain core.BlockChain
headerRepository datastore.HeaderRepository
windowSize int
transformers []transformers.Transformer
}
func NewHeaderValidator(blockChain core.BlockChain, repository datastore.HeaderRepository, windowSize int, transformers []transformers.Transformer) HeaderValidator {
func NewHeaderValidator(blockChain core.BlockChain, repository datastore.HeaderRepository, windowSize int) HeaderValidator {
return HeaderValidator{
blockChain: blockChain,
headerRepository: repository,
windowSize: windowSize,
transformers: transformers,
}
}
func (validator HeaderValidator) ValidateHeaders() ValidationWindow {
window := MakeValidationWindow(validator.blockChain, validator.windowSize)
blockNumbers := MakeRange(window.LowerBound, window.UpperBound)
RetrieveAndUpdateHeaders(validator.blockChain, validator.headerRepository, blockNumbers, validator.transformers)
RetrieveAndUpdateHeaders(validator.blockChain, validator.headerRepository, blockNumbers)
return window
}

18
pkg/history/header_validator_test.go
View File

@ -2,10 +2,8 @@ package history_test
import (
. "github.com/onsi/ginkgo"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/history"
"github.com/vulcanize/vulcanizedb/pkg/transformers"
"math/big"
)
@ -15,24 +13,10 @@ var _ = Describe("Header validator", func() {
headerRepository.SetMissingBlockNumbers([]int64{})
blockChain := fakes.NewMockBlockChain()
blockChain.SetLastBlock(big.NewInt(3))
validator := history.NewHeaderValidator(blockChain, headerRepository, 2, []transformers.Transformer{})
validator := history.NewHeaderValidator(blockChain, headerRepository, 2)
validator.ValidateHeaders()
headerRepository.AssertCreateOrUpdateHeaderCallCountAndPassedBlockNumbers(3, []int64{1, 2, 3})
})
It("passes transformers for execution on new blocks", func() {
headerRepository := fakes.NewMockHeaderRepository()
headerRepository.SetMissingBlockNumbers([]int64{})
blockChain := fakes.NewMockBlockChain()
blockChain.SetLastBlock(big.NewInt(3))
transformer := fakes.NewMockTransformer()
validator := history.NewHeaderValidator(blockChain, headerRepository, 1, []transformers.Transformer{transformer})
validator.ValidateHeaders()
headerRepository.AssertCreateOrUpdateHeaderCallCountAndPassedBlockNumbers(2, []int64{2, 3})
transformer.AssertExecuteCalledWith(core.Header{BlockNumber: 3}, 0)
})
})

15
pkg/history/populate_headers.go
View File

@ -6,40 +6,33 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/transformers"
)
func PopulateMissingHeaders(blockchain core.BlockChain, headerRepository datastore.HeaderRepository, startingBlockNumber int64, transformers []transformers.Transformer) (int, error) {
func PopulateMissingHeaders(blockchain core.BlockChain, headerRepository datastore.HeaderRepository, startingBlockNumber int64) (int, error) {
lastBlock := blockchain.LastBlock().Int64()
blockRange := headerRepository.MissingBlockNumbers(startingBlockNumber, lastBlock, blockchain.Node().ID)
log.SetPrefix("")
log.Printf("Backfilling %d blocks\n\n", len(blockRange))
_, err := RetrieveAndUpdateHeaders(blockchain, headerRepository, blockRange, transformers)
_, err := RetrieveAndUpdateHeaders(blockchain, headerRepository, blockRange)
if err != nil {
return 0, err
}
return len(blockRange), nil
}
func RetrieveAndUpdateHeaders(chain core.BlockChain, headerRepository datastore.HeaderRepository, blockNumbers []int64, transformers []transformers.Transformer) (int, error) {
func RetrieveAndUpdateHeaders(chain core.BlockChain, headerRepository datastore.HeaderRepository, blockNumbers []int64) (int, error) {
for _, blockNumber := range blockNumbers {
header, err := chain.GetHeaderByNumber(blockNumber)
if err != nil {
return 0, err
}
id, err := headerRepository.CreateOrUpdateHeader(header)
_, err = headerRepository.CreateOrUpdateHeader(header)
if err != nil {
if err == repositories.ErrValidHeaderExists {
continue
}
return 0, err
}
for _, transformer := range transformers {
err := transformer.Execute(header, id)
if err != nil {
return 0, err
}
}
}
return len(blockNumbers), nil
}

52
pkg/history/populate_headers_test.go
View File

@ -6,11 +6,8 @@ import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/history"
"github.com/vulcanize/vulcanizedb/pkg/transformers"
)
var _ = Describe("Populating headers", func() {
@ -26,7 +23,7 @@ var _ = Describe("Populating headers", func() {
blockChain.SetLastBlock(big.NewInt(2))
headerRepository.SetMissingBlockNumbers([]int64{2})
headersAdded, err := history.PopulateMissingHeaders(blockChain, headerRepository, 1, []transformers.Transformer{})
headersAdded, err := history.PopulateMissingHeaders(blockChain, headerRepository, 1)
Expect(err).NotTo(HaveOccurred())
Expect(headersAdded).To(Equal(1))
@ -37,54 +34,9 @@ var _ = Describe("Populating headers", func() {
blockChain.SetLastBlock(big.NewInt(2))
headerRepository.SetMissingBlockNumbers([]int64{2})
_, err := history.PopulateMissingHeaders(blockChain, headerRepository, 1, []transformers.Transformer{})
_, err := history.PopulateMissingHeaders(blockChain, headerRepository, 1)
Expect(err).NotTo(HaveOccurred())
headerRepository.AssertCreateOrUpdateHeaderCallCountAndPassedBlockNumbers(1, []int64{2})
})
It("executes passed transformers with created headers", func() {
blockNumber := int64(54321)
blockChain := fakes.NewMockBlockChain()
blockChain.SetLastBlock(big.NewInt(blockNumber))
headerRepository.SetMissingBlockNumbers([]int64{blockNumber})
headerID := int64(12345)
headerRepository.SetCreateOrUpdateHeaderReturnID(headerID)
transformer := fakes.NewMockTransformer()
_, err := history.PopulateMissingHeaders(blockChain, headerRepository, blockNumber, []transformers.Transformer{transformer})
Expect(err).NotTo(HaveOccurred())
transformer.AssertExecuteCalledWith(core.Header{BlockNumber: blockNumber}, headerID)
})
It("does not execute transformer if repository indicates header already exists", func() {
blockNumber := int64(54321)
blockChain := fakes.NewMockBlockChain()
blockChain.SetLastBlock(big.NewInt(blockNumber))
headerRepository.SetMissingBlockNumbers([]int64{blockNumber})
headerRepository.SetCreateOrUpdateHeaderReturnErr(repositories.ErrValidHeaderExists)
transformer := fakes.NewMockTransformer()
_, err := history.PopulateMissingHeaders(blockChain, headerRepository, blockNumber, []transformers.Transformer{transformer})
Expect(err).NotTo(HaveOccurred())
transformer.AssertExecuteNotCalled()
})
It("returns error if executing transformer fails", func() {
blockNumber := int64(54321)
blockChain := fakes.NewMockBlockChain()
blockChain.SetLastBlock(big.NewInt(blockNumber))
headerRepository.SetMissingBlockNumbers([]int64{blockNumber})
headerID := int64(12345)
headerRepository.SetCreateOrUpdateHeaderReturnID(headerID)
transformer := fakes.NewMockTransformer()
transformer.SetExecuteErr(fakes.FakeError)
_, err := history.PopulateMissingHeaders(blockChain, headerRepository, blockNumber, []transformers.Transformer{transformer})
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
})

12
pkg/transformers/flip_kick/config.go
View File

@ -14,16 +14,10 @@
package flip_kick
type TransformerConfig struct {
ContractAddress string
ContractAbi string
Topics []string
StartingBlockNumber int64
EndingBlockNumber int64
}
import "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
var FlipKickConfig = TransformerConfig{
ContractAddress: "0x08cb6176addcca2e1d1ffe21bee464b72ee4cd8d", //this is a temporary address deployed locally
var FlipKickConfig = shared.TransformerConfig{
ContractAddresses: "0x08cb6176addcca2e1d1ffe21bee464b72ee4cd8d", //this is a temporary address deployed locally
ContractAbi: FlipperABI,
Topics: []string{FlipKickSignature},
StartingBlockNumber: 0,

10
pkg/transformers/flip_kick/transformer.go
View File

@ -30,11 +30,11 @@ type FlipKickTransformer struct {
Fetcher shared.LogFetcher
Converter Converter
Repository Repository
Config TransformerConfig
Config shared.TransformerConfig
}
type FlipKickTransformerInitializer struct {
Config TransformerConfig
Config shared.TransformerConfig
}
func (i FlipKickTransformerInitializer) NewFlipKickTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer {
@ -50,7 +50,7 @@ func (i FlipKickTransformerInitializer) NewFlipKickTransformer(db *postgres.DB,
return transformer
}
func (fkt *FlipKickTransformer) SetConfig(config TransformerConfig) {
func (fkt *FlipKickTransformer) SetConfig(config shared.TransformerConfig) {
fkt.Config = config
}
@ -91,13 +91,13 @@ func (fkt FlipKickTransformer) Execute() error {
log.Printf("Fetching event logs for %d headers \n", len(headers))
var resultingErrors []error
for _, header := range headers {
ethLogs, err := fkt.Fetcher.FetchLogs(config.ContractAddress, topics, header.BlockNumber)
ethLogs, err := fkt.Fetcher.FetchLogs(config.ContractAddresses, topics, header.BlockNumber)
if err != nil {
resultingErrors = append(resultingErrors, newTransformerError(err, header.BlockNumber, FetcherError))
}
for _, ethLog := range ethLogs {
entity, err := fkt.Converter.ToEntity(config.ContractAddress, config.ContractAbi, ethLog)
entity, err := fkt.Converter.ToEntity(config.ContractAddresses, config.ContractAbi, ethLog)
if err != nil {
resultingErrors = append(resultingErrors, newTransformerError(err, header.BlockNumber, LogToEntityError))
}

11
pkg/transformers/flip_kick/transformer_test.go
View File

@ -25,6 +25,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/transformers/flip_kick"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks"
flip_kick_mocks "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks/flip_kick"
@ -35,7 +36,7 @@ var _ = Describe("FlipKick Transformer", func() {
var fetcher mocks.MockLogFetcher
var converter flip_kick_mocks.MockFlipKickConverter
var repository flip_kick_mocks.MockFlipKickRepository
var testConfig flip_kick.TransformerConfig
var testConfig shared.TransformerConfig
var blockNumber int64
var headerId int64
var headers []core.Header
@ -52,8 +53,8 @@ var _ = Describe("FlipKick Transformer", func() {
}
startingBlockNumber := rand.Int63()
testConfig = flip_kick.TransformerConfig{
ContractAddress: "0x12345",
testConfig = shared.TransformerConfig{
ContractAddresses: "0x12345",
ContractAbi: "test abi",
Topics: []string{flip_kick.FlipKickSignature},
StartingBlockNumber: startingBlockNumber,
@ -82,7 +83,7 @@ var _ = Describe("FlipKick Transformer", func() {
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(fetcher.FetchedContractAddress).To(Equal(testConfig.ContractAddress))
Expect(fetcher.FetchedContractAddress).To(Equal(testConfig.ContractAddresses))
Expect(fetcher.FetchedTopics).To(Equal(expectedTopics))
Expect(fetcher.FetchedBlocks).To(Equal([]int64{blockNumber}))
})
@ -99,7 +100,7 @@ var _ = Describe("FlipKick Transformer", func() {
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(converter.ConverterContract).To(Equal(testConfig.ContractAddress))
Expect(converter.ConverterContract).To(Equal(testConfig.ContractAddresses))
Expect(converter.ConverterAbi).To(Equal(testConfig.ContractAbi))
Expect(converter.LogsToConvert).To(Equal(logs))
Expect(converter.EntitiesToConvert).To(Equal([]flip_kick.FlipKickEntity{test_data.FlipKickEntity}))

12
pkg/transformers/frob/config.go
View File

@ -14,16 +14,10 @@
package frob
type TransformerConfig struct {
ContractAddress string
ContractAbi string
Topics []string
StartingBlockNumber int64
EndingBlockNumber int64
}
import "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
var FrobConfig = TransformerConfig{
ContractAddress: "0xff3f2400f1600f3f493a9a92704a29b96795af1a", //this is a temporary address deployed locally
var FrobConfig = shared.TransformerConfig{
ContractAddresses: "0xff3f2400f1600f3f493a9a92704a29b96795af1a", //this is a temporary address deployed locally
ContractAbi: FrobABI,
Topics: []string{FrobEventSignature},
StartingBlockNumber: 0,

10
pkg/transformers/frob/transformer.go
View File

@ -23,13 +23,14 @@ import (
)
type FrobTransformer struct {
Config shared.TransformerConfig
Converter Converter
Fetcher shared.LogFetcher
Repository Repository
}
type FrobTransformerInitializer struct {
Config TransformerConfig
Config shared.TransformerConfig
}
func (initializer FrobTransformerInitializer) NewFrobTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer {
@ -37,6 +38,7 @@ func (initializer FrobTransformerInitializer) NewFrobTransformer(db *postgres.DB
fetcher := shared.NewFetcher(blockChain)
repository := NewFrobRepository(db)
return FrobTransformer{
Config: initializer.Config,
Converter: converter,
Fetcher: fetcher,
Repository: repository,
@ -44,18 +46,18 @@ func (initializer FrobTransformerInitializer) NewFrobTransformer(db *postgres.DB
}
func (transformer FrobTransformer) Execute() error {
missingHeaders, err := transformer.Repository.MissingHeaders(FrobConfig.StartingBlockNumber, FrobConfig.EndingBlockNumber)
missingHeaders, err := transformer.Repository.MissingHeaders(transformer.Config.StartingBlockNumber, transformer.Config.EndingBlockNumber)
if err != nil {
return err
}
for _, header := range missingHeaders {
topics := [][]common.Hash{{common.HexToHash(FrobEventSignature)}}
matchingLogs, err := transformer.Fetcher.FetchLogs(FrobConfig.ContractAddress, topics, header.BlockNumber)
matchingLogs, err := transformer.Fetcher.FetchLogs(FrobConfig.ContractAddresses, topics, header.BlockNumber)
if err != nil {
return err
}
for _, log := range matchingLogs {
entity, err := transformer.Converter.ToEntity(FrobConfig.ContractAddress, FrobConfig.ContractAbi, log)
entity, err := transformer.Converter.ToEntity(FrobConfig.ContractAddresses, FrobConfig.ContractAbi, log)
if err != nil {
return err
}

5
pkg/transformers/frob/transformer_test.go
View File

@ -32,6 +32,7 @@ var _ = Describe("Frob transformer", func() {
It("gets missing headers for block numbers specified in config", func() {
repository := &frob_mocks.MockFrobRepository{}
transformer := frob.FrobTransformer{
Config: frob.FrobConfig,
Fetcher: &mocks.MockLogFetcher{},
Converter: &frob_mocks.MockFrobConverter{},
Repository: repository,
@ -73,7 +74,7 @@ var _ = Describe("Frob transformer", func() {
Expect(err).NotTo(HaveOccurred())
Expect(fetcher.FetchedBlocks).To(Equal([]int64{1, 2}))
Expect(fetcher.FetchedContractAddress).To(Equal(frob.FrobConfig.ContractAddress))
Expect(fetcher.FetchedContractAddress).To(Equal(frob.FrobConfig.ContractAddresses))
Expect(fetcher.FetchedTopics).To(Equal([][]common.Hash{{common.HexToHash(frob.FrobEventSignature)}}))
})
@ -109,7 +110,7 @@ var _ = Describe("Frob transformer", func() {
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(converter.PassedContractAddress).To(Equal(frob.FrobConfig.ContractAddress))
Expect(converter.PassedContractAddress).To(Equal(frob.FrobConfig.ContractAddresses))
Expect(converter.PassedContractABI).To(Equal(frob.FrobConfig.ContractAbi))
Expect(converter.PassedLog).To(Equal(test_data.EthFrobLog))
Expect(converter.PassedEntity).To(Equal(test_data.FrobEntity))

37
pkg/transformers/price_feeds/config.go Normal file
View File

@ -0,0 +1,37 @@
// Copyright © 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package price_feeds
var (
PepAddress = "0x99041F808D598B782D5a3e498681C2452A31da08"
PipAddress = "0x729D19f657BD0614b4985Cf1D82531c67569197B"
RepAddress = "0xF5f94b7F9De14D43112e713835BCef2d55b76c1C"
)
type IPriceFeedConfig struct {
ContractAddresses []string
StartingBlockNumber int64
EndingBlockNumber int64
}
var PriceFeedConfig = IPriceFeedConfig{
ContractAddresses: []string{
PepAddress,
PipAddress,
RepAddress,
},
StartingBlockNumber: 0,
EndingBlockNumber: 100,
}

22
pkg/transformers/price_feeds/constants.go
View File

@ -1,3 +1,17 @@
// Copyright © 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package price_feeds
import (
@ -6,13 +20,9 @@ import (
)
var (
ErrMultipleLogs = errors.New("multiple matching logs found in block")
ErrNoMatchingLog = errors.New("no matching log")
Ether = big.NewFloat(1e18)
PeekMethodName = "peek"
PepLogTopic0 = "0x296ba4ca62c6c21c95e828080cb8aec7481b71390585605300a8a76f9e95b527"
PipLogTopic0 = "0x1817835800000000000000000000000000000000000000000000000000000000"
PipMedianizerABI = `[{"constant":false,"inputs":[{"name":"owner_","type":"address"}],"name":"setOwner","outputs":[],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"","type":"bytes32"}],"name":"poke","outputs":[],"payable":false,"type":"function"},{"constant":false,"inputs":[],"name":"poke","outputs":[],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"compute","outputs":[{"name":"","type":"bytes32"},{"name":"","type":"bool"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"wat","type":"address"}],"name":"set","outputs":[],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"wat","type":"address"}],"name":"unset","outputs":[],"payable":false,"type":"function"},{"constant":true,"inputs":[{"name":"","type":"address"}],"name":"indexes","outputs":[{"name":"","type":"bytes12"}],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"next","outputs":[{"name":"","type":"bytes12"}],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"read","outputs":[{"name":"","type":"bytes32"}],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"peek","outputs":[{"name":"","type":"bytes32"},{"name":"","type":"bool"}],"payable":false,"type":"function"},{"constant":true,"inputs":[{"name":"","type":"bytes12"}],"name":"values","outputs":[{"name":"","type":"address"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"min_","type":"uint96"}],"name":"setMin","outputs":[],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"authority_","type":"address"}],"name":"setAuthority","outputs":[],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"owner","outputs":[{"name":"","type":"address"}],"payable":false,"type":"function"},{"constant":false,"inputs":[],"name":"void","outputs":[],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"pos","type":"bytes12"},{"name":"wat","type":"address"}],"name":"set","outputs":[],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"authority","outputs":[{"name":"","type":"address"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"pos","type":"bytes12"}],"name":"unset","outputs":[],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"next_","type":"bytes12"}],"name":"setNext","outputs":[],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"min","outputs":[{"name":"","type":"uint96"}],"payable":false,"type":"function"},{"anonymous":true,"inputs":[{"indexed":true,"name":"sig","type":"bytes4"},{"indexed":true,"name":"guy","type":"address"},{"indexed":true,"name":"foo","type":"bytes32"},{"indexed":true,"name":"bar","type":"bytes32"},{"indexed":false,"name":"wad","type":"uint256"},{"indexed":false,"name":"fax","type":"bytes"}],"name":"LogNote","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"authority","type":"address"}],"name":"LogSetAuthority","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"owner","type":"address"}],"name":"LogSetOwner","type":"event"}]`
LogValueTopic0 = "0x296ba4ca62c6c21c95e828080cb8aec7481b71390585605300a8a76f9e95b527"
MedianizerABI = `[{"constant":false,"inputs":[{"name":"owner_","type":"address"}],"name":"setOwner","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"","type":"bytes32"}],"name":"poke","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[],"name":"poke","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"compute","outputs":[{"name":"","type":"bytes32"},{"name":"","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"wat","type":"address"}],"name":"set","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"wat","type":"address"}],"name":"unset","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[{"name":"","type":"address"}],"name":"indexes","outputs":[{"name":"","type":"bytes12"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"next","outputs":[{"name":"","type":"bytes12"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"read","outputs":[{"name":"","type":"bytes32"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"peek","outputs":[{"name":"","type":"bytes32"},{"name":"","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"","type":"bytes12"}],"name":"values","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"min_","type":"uint96"}],"name":"setMin","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"authority_","type":"address"}],"name":"setAuthority","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"owner","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[],"name":"void","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"pos","type":"bytes12"},{"name":"wat","type":"address"}],"name":"set","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"authority","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"pos","type":"bytes12"}],"name":"unset","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"next_","type":"bytes12"}],"name":"setNext","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"min","outputs":[{"name":"","type":"uint96"}],"payable":false,"stateMutability":"view","type":"function"},{"anonymous":false,"inputs":[{"indexed":false,"name":"val","type":"bytes32"}],"name":"LogValue","type":"event"},{"anonymous":true,"inputs":[{"indexed":true,"name":"sig","type":"bytes4"},{"indexed":true,"name":"guy","type":"address"},{"indexed":true,"name":"foo","type":"bytes32"},{"indexed":true,"name":"bar","type":"bytes32"},{"indexed":false,"name":"wad","type":"uint256"},{"indexed":false,"name":"fax","type":"bytes"}],"name":"LogNote","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"authority","type":"address"}],"name":"LogSetAuthority","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"owner","type":"address"}],"name":"LogSetOwner","type":"event"}]]`
Ray = big.NewFloat(1e27)
RepLogTopic0 = "0x296ba4ca62c6c21c95e828080cb8aec7481b71390585605300a8a76f9e95b527"
)

32
pkg/transformers/price_feeds/converter.go Normal file
View File

@ -0,0 +1,32 @@
// Copyright © 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package price_feeds
import (
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/core/types"
)
type PriceFeedConverter struct{}
func (converter PriceFeedConverter) ToModel(log types.Log, headerID int64) PriceFeedModel {
return PriceFeedModel{
BlockNumber: log.BlockNumber,
HeaderID: headerID,
MedianizerAddress: log.Address.Bytes(),
UsdValue: Convert("wad", hexutil.Encode(log.Data), 15),
TransactionIndex: log.TxIndex,
}
}

57
pkg/transformers/price_feeds/converter_test.go Normal file
View File

@ -0,0 +1,57 @@
// Copyright © 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package price_feeds_test
import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds"
)
var _ = Describe("Price feed converter", func() {
It("converts a log to a price feed model", func() {
medianizerAddress := common.HexToAddress("0x99041f808d598b782d5a3e498681c2452a31da08")
blockNumber := uint64(6147230)
txIndex := uint(119)
// https://etherscan.io/tx/0xa51a50a2adbfba4e2ab3d72dfd67a21c769f1bc8d2b180663a15500a56cde58f
log := types.Log{
Address: medianizerAddress,
Topics: []common.Hash{common.HexToHash(price_feeds.LogValueTopic0)},
Data: common.FromHex("00000000000000000000000000000000000000000000001486f658319fb0c100"),
BlockNumber: blockNumber,
TxHash: common.HexToHash("0xa51a50a2adbfba4e2ab3d72dfd67a21c769f1bc8d2b180663a15500a56cde58f"),
TxIndex: txIndex,
BlockHash: common.HexToHash("0x27ecebbf69eefa3bb3cf65f472322a80ff4946653a50a2171dc605f49829467d"),
Index: 0,
Removed: false,
}
converter := price_feeds.PriceFeedConverter{}
headerID := int64(123)
model := converter.ToModel(log, headerID)
expectedModel := price_feeds.PriceFeedModel{
BlockNumber: blockNumber,
HeaderID: headerID,
MedianizerAddress: medianizerAddress[:],
UsdValue: "378.6599388897",
TransactionIndex: txIndex,
}
Expect(model).To(Equal(expectedModel))
})
})

54
pkg/transformers/price_feeds/fetcher.go Normal file
View File

@ -0,0 +1,54 @@
// Copyright © 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package price_feeds
import (
"github.com/ethereum/go-ethereum"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/core"
"math/big"
)
type IPriceFeedFetcher interface {
FetchLogValues(blockNumber int64) ([]types.Log, error)
}
type PriceFeedFetcher struct {
blockChain core.BlockChain
contractAddresses []string
}
func NewPriceFeedFetcher(blockChain core.BlockChain, contractAddresses []string) PriceFeedFetcher {
return PriceFeedFetcher{
blockChain: blockChain,
contractAddresses: contractAddresses,
}
}
func (fetcher PriceFeedFetcher) FetchLogValues(blockNumber int64) ([]types.Log, error) {
var addresses []common.Address
for _, addr := range fetcher.contractAddresses {
addresses = append(addresses, common.HexToAddress(addr))
}
n := big.NewInt(blockNumber)
query := ethereum.FilterQuery{
FromBlock: n,
ToBlock: n,
Addresses: addresses,
Topics: [][]common.Hash{{common.HexToHash(LogValueTopic0)}},
}
return fetcher.blockChain.GetEthLogsWithCustomQuery(query)
}

64
pkg/transformers/price_feeds/fetcher_test.go Normal file
View File

@ -0,0 +1,64 @@
// Copyright © 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package price_feeds_test
import (
"math/big"
"github.com/ethereum/go-ethereum"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds"
)
var _ = Describe("Price fetcher", func() {
It("gets log value events from price feed medianizers", func() {
mockBlockChain := fakes.NewMockBlockChain()
mockBlockChain.SetGetEthLogsWithCustomQueryReturnLogs([]types.Log{{}})
contractAddresses := []string{"pep-contract-address", "pip-contract-address", "rep-contract-address"}
fetcher := price_feeds.NewPriceFeedFetcher(mockBlockChain, contractAddresses)
blockNumber := int64(100)
_, err := fetcher.FetchLogValues(blockNumber)
Expect(err).NotTo(HaveOccurred())
var expectedAddresses []common.Address
for _, address := range contractAddresses {
expectedAddresses = append(expectedAddresses, common.HexToAddress(address))
}
expectedQuery := ethereum.FilterQuery{
FromBlock: big.NewInt(blockNumber),
ToBlock: big.NewInt(blockNumber),
Addresses: expectedAddresses,
Topics: [][]common.Hash{{common.HexToHash(price_feeds.LogValueTopic0)}},
}
mockBlockChain.AssertGetEthLogsWithCustomQueryCalledWith(expectedQuery)
})
It("returns error if getting logs fails", func() {
mockBlockChain := fakes.NewMockBlockChain()
mockBlockChain.SetGetEthLogsWithCustomQueryErr(fakes.FakeError)
fetcher := price_feeds.NewPriceFeedFetcher(mockBlockChain, []string{"contract-address"})
_, err := fetcher.FetchLogValues(100)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
})

53
pkg/transformers/price_feeds/pep/fetcher.go
View File

@ -1,53 +0,0 @@
package pep
import (
"math/big"
"github.com/ethereum/go-ethereum"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds"
)
type IPepFetcher interface {
FetchPepValue(header core.Header) (string, error)
}
type PepFetcher struct {
blockChain core.BlockChain
contractAddress string
}
func NewPepFetcher(chain core.BlockChain, contractAddress string) PepFetcher {
return PepFetcher{
blockChain: chain,
contractAddress: contractAddress,
}
}
func (fetcher PepFetcher) FetchPepValue(header core.Header) (string, error) {
blockNumber := big.NewInt(header.BlockNumber)
query := ethereum.FilterQuery{
FromBlock: blockNumber,
ToBlock: blockNumber,
Addresses: []common.Address{common.HexToAddress(fetcher.contractAddress)},
Topics: [][]common.Hash{{common.HexToHash(price_feeds.PepLogTopic0)}},
}
logs, err := fetcher.blockChain.GetEthLogsWithCustomQuery(query)
return fetcher.getLogValue(logs, err)
}
func (fetcher PepFetcher) getLogValue(logs []types.Log, err error) (string, error) {
if err != nil {
return "", err
}
if len(logs) < 1 {
return "", price_feeds.ErrNoMatchingLog
}
if len(logs) > 1 {
return "", price_feeds.ErrMultipleLogs
}
return string(logs[0].Data), nil
}

74
pkg/transformers/price_feeds/pep/fetcher_test.go
View File

@ -1,74 +0,0 @@
package pep_test
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/ethereum/go-ethereum"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds"
"github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds/pep"
"math/big"
)
var _ = Describe("Pep fetcher", func() {
It("gets logs describing updated mkr/usd value", func() {
mockBlockChain := fakes.NewMockBlockChain()
mockBlockChain.SetGetEthLogsWithCustomQueryReturnLogs([]types.Log{{}})
contractAddress := "pep-contract-address"
fetcher := pep.NewPepFetcher(mockBlockChain, contractAddress)
blockNumber := int64(12345)
header := core.Header{
BlockNumber: blockNumber,
Hash: "",
Raw: nil,
}
_, err := fetcher.FetchPepValue(header)
Expect(err).NotTo(HaveOccurred())
expectedQuery := ethereum.FilterQuery{
FromBlock: big.NewInt(blockNumber),
ToBlock: big.NewInt(blockNumber),
Addresses: []common.Address{common.HexToAddress(contractAddress)},
Topics: [][]common.Hash{{common.HexToHash(price_feeds.PepLogTopic0)}},
}
mockBlockChain.AssertGetEthLogsWithCustomQueryCalledWith(expectedQuery)
})
It("returns error if getting logs fails", func() {
mockBlockChain := fakes.NewMockBlockChain()
mockBlockChain.SetGetEthLogsWithCustomQueryReturnLogs([]types.Log{{}})
mockBlockChain.SetGetEthLogsWithCustomQueryErr(fakes.FakeError)
fetcher := pep.NewPepFetcher(mockBlockChain, "pep-contract-address")
_, err := fetcher.FetchPepValue(core.Header{})
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("returns no matching logs error if no logs returned", func() {
mockBlockChain := fakes.NewMockBlockChain()
fetcher := pep.NewPepFetcher(mockBlockChain, "pep-contract-address")
_, err := fetcher.FetchPepValue(core.Header{})
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(price_feeds.ErrNoMatchingLog))
})
It("returns error if more than one matching logs returned", func() {
mockBlockChain := fakes.NewMockBlockChain()
mockBlockChain.SetGetEthLogsWithCustomQueryReturnLogs([]types.Log{{}, {}})
fetcher := pep.NewPepFetcher(mockBlockChain, "pep-contract-address")
_, err := fetcher.FetchPepValue(core.Header{})
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(price_feeds.ErrMultipleLogs))
})
})

13
pkg/transformers/price_feeds/pep/pep_suite_test.go
View File

@ -1,13 +0,0 @@
package pep
import (
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
func TestPep(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Pep Suite")
}

25
pkg/transformers/price_feeds/pep/repository.go
View File

@ -1,25 +0,0 @@
package pep
import (
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds"
)
type IPepRepository interface {
CreatePep(pep price_feeds.PriceUpdate) error
}
type PepRepository struct {
db *postgres.DB
}
func NewPepRepository(db *postgres.DB) PepRepository {
return PepRepository{
db: db,
}
}
func (repository PepRepository) CreatePep(pep price_feeds.PriceUpdate) error {
_, err := repository.db.Exec(`INSERT INTO maker.peps (block_number, header_id, usd_value) VALUES ($1, $2, $3::NUMERIC)`, pep.BlockNumber, pep.HeaderID, pep.UsdValue)
return err
}

51
pkg/transformers/price_feeds/pep/repository_test.go
View File

@ -1,51 +0,0 @@
package pep_test
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds"
"github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds/pep"
"github.com/vulcanize/vulcanizedb/test_config"
)
var _ = Describe("Pep repository", func() {
It("returns header if matching header does not exist", func() {
db := test_config.NewTestDB(core.Node{})
repository := pep.NewPepRepository(db)
pepToAdd := price_feeds.PriceUpdate{
BlockNumber: 0,
HeaderID: 0,
UsdValue: "123.456",
}
err := repository.CreatePep(pepToAdd)
Expect(err).To(HaveOccurred())
})
It("creates a pep when matching header exists", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
repository := pep.NewPepRepository(db)
header := core.Header{BlockNumber: 12345}
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(header)
Expect(err).NotTo(HaveOccurred())
pepToAdd := price_feeds.PriceUpdate{
BlockNumber: header.BlockNumber,
HeaderID: headerID,
UsdValue: "123.456",
}
err = repository.CreatePep(pepToAdd)
Expect(err).NotTo(HaveOccurred())
var dbPep price_feeds.PriceUpdate
err = db.Get(&dbPep, `SELECT block_number, header_id, usd_value FROM maker.peps WHERE header_id = $1`, pepToAdd.HeaderID)
Expect(err).NotTo(HaveOccurred())
Expect(dbPep).To(Equal(pepToAdd))
})
})

43
pkg/transformers/price_feeds/pep/transformer.go
View File

@ -1,43 +0,0 @@
package pep
import (
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds"
)
type PepTransformer struct {
fetcher IPepFetcher
repository IPepRepository
}
func NewPepTransformer(chain core.BlockChain, db *postgres.DB, contractAddress string) PepTransformer {
fetcher := NewPepFetcher(chain, contractAddress)
repository := NewPepRepository(db)
return PepTransformer{
fetcher: fetcher,
repository: repository,
}
}
func (transformer PepTransformer) Execute(header core.Header, headerID int64) error {
logValue, err := transformer.fetcher.FetchPepValue(header)
if err != nil {
if err == price_feeds.ErrNoMatchingLog {
return nil
}
return err
}
pep := getPep(logValue, header, headerID)
return transformer.repository.CreatePep(pep)
}
func getPep(logValue string, header core.Header, headerID int64) price_feeds.PriceUpdate {
valueInUSD := price_feeds.Convert("wad", logValue, 15)
pep := price_feeds.PriceUpdate{
BlockNumber: header.BlockNumber,
HeaderID: headerID,
UsdValue: valueInUSD,
}
return pep
}

46
pkg/transformers/price_feeds/pep/transformer_test.go
View File

@ -1,46 +0,0 @@
package pep_test
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds"
"github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds/pep"
"github.com/vulcanize/vulcanizedb/test_config"
)
var _ = Describe("Pep transformer", func() {
It("returns nil if no logs found", func() {
chain := fakes.NewMockBlockChain()
db := test_config.NewTestDB(core.Node{})
transformer := pep.NewPepTransformer(chain, db, "pep-contract-address")
err := transformer.Execute(core.Header{}, 123)
Expect(err).NotTo(HaveOccurred())
})
It("creates pep row for found log", func() {
chain := fakes.NewMockBlockChain()
chain.SetGetEthLogsWithCustomQueryReturnLogs([]types.Log{{Data: []byte{1, 2, 3, 4, 5}}})
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
header := core.Header{BlockNumber: 12345}
headerID, err := headerRepository.CreateOrUpdateHeader(header)
Expect(err).NotTo(HaveOccurred())
transformer := pep.NewPepTransformer(chain, db, "pep-contract-address")
err = transformer.Execute(header, headerID)
Expect(err).NotTo(HaveOccurred())
var dbPep price_feeds.PriceUpdate
err = db.Get(&dbPep, `SELECT block_number, header_id, usd_value FROM maker.peps WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(dbPep.BlockNumber).To(Equal(header.BlockNumber))
})
})

77
pkg/transformers/price_feeds/pip/fetcher.go
View File

@ -1,77 +0,0 @@
package pip
import (
"github.com/ethereum/go-ethereum"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds"
"math/big"
)
type IPipFetcher interface {
FetchPipValue(header core.Header) (string, error)
}
type PipFetcher struct {
blockChain core.BlockChain
contractAddress string
}
func NewPipFetcher(chain core.BlockChain, contractAddress string) PipFetcher {
return PipFetcher{
blockChain: chain,
contractAddress: contractAddress,
}
}
func (fetcher PipFetcher) FetchPipValue(header core.Header) (string, error) {
blockNumber := big.NewInt(header.BlockNumber)
query := ethereum.FilterQuery{
FromBlock: blockNumber,
ToBlock: blockNumber,
Addresses: []common.Address{common.HexToAddress(fetcher.contractAddress)},
Topics: [][]common.Hash{{common.HexToHash(price_feeds.PipLogTopic0)}},
}
logs, err := fetcher.blockChain.GetEthLogsWithCustomQuery(query)
if err != nil {
return "", err
}
if len(logs) > 0 {
return fetcher.getLogValue(logs, err)
}
return "", price_feeds.ErrNoMatchingLog
}
func (fetcher PipFetcher) getLogValue(logs []types.Log, err error) (string, error) {
var (
ret0 = new([32]byte)
ret1 = new(bool)
)
var r = &[]interface{}{
ret0,
ret1,
}
err = fetcher.blockChain.FetchContractData(price_feeds.PipMedianizerABI, fetcher.contractAddress, price_feeds.PeekMethodName, nil, r, int64(logs[0].BlockNumber))
if err != nil {
return "", err
}
result := newResult(*ret0, *ret1)
return result.Value.String(), nil
}
type Value [32]byte
type Peek struct {
Value
OK bool
}
func (value Value) String() string {
bi := big.NewInt(0).SetBytes(value[:])
return bi.String()
}
func newResult(value [32]byte, ok bool) *Peek {
return &Peek{Value: value, OK: ok}
}

61
pkg/transformers/price_feeds/pip/fetcher_test.go
View File

@ -1,61 +0,0 @@
package pip_test
import (
"github.com/ethereum/go-ethereum/core/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds"
"github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds/pip"
)
var _ = Describe("Pip fetcher", func() {
It("returns error if fetching logs fails", func() {
chain := fakes.NewMockBlockChain()
chain.SetGetEthLogsWithCustomQueryErr(fakes.FakeError)
fetcher := pip.NewPipFetcher(chain, "pip-contract-address")
_, err := fetcher.FetchPipValue(core.Header{})
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("returns no matching logs error if no logs returned", func() {
chain := fakes.NewMockBlockChain()
fetcher := pip.NewPipFetcher(chain, "pip-contract-address")
_, err := fetcher.FetchPipValue(core.Header{})
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(price_feeds.ErrNoMatchingLog))
})
Describe("when matching log found", func() {
It("calls contract to peek current eth/usd value", func() {
blockNumber := uint64(12345)
chain := fakes.NewMockBlockChain()
chain.SetGetEthLogsWithCustomQueryReturnLogs([]types.Log{{BlockNumber: blockNumber}})
contractAddress := "pip-contract-address"
fetcher := pip.NewPipFetcher(chain, contractAddress)
_, err := fetcher.FetchPipValue(core.Header{})
Expect(err).NotTo(HaveOccurred())
chain.AssertFetchContractDataCalledWith(price_feeds.PipMedianizerABI, contractAddress, price_feeds.PeekMethodName, nil, &[]interface{}{[32]byte{}, false}, int64(blockNumber))
})
It("returns error if contract call fails", func() {
chain := fakes.NewMockBlockChain()
chain.SetGetEthLogsWithCustomQueryReturnLogs([]types.Log{{BlockNumber: uint64(12345)}})
chain.SetFetchContractDataErr(fakes.FakeError)
fetcher := pip.NewPipFetcher(chain, "pip-contract-address")
_, err := fetcher.FetchPipValue(core.Header{})
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
})
})

13
pkg/transformers/price_feeds/pip/pip_suite_test.go
View File

@ -1,13 +0,0 @@
package pip
import (
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
func TestPip(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Pip Suite")
}

25
pkg/transformers/price_feeds/pip/repository.go
View File

@ -1,25 +0,0 @@
package pip
import (
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds"
)
type IPipRepository interface {
CreatePip(pip price_feeds.PriceUpdate) error
}
type PipRepository struct {
db *postgres.DB
}
func NewPipRepository(db *postgres.DB) PipRepository {
return PipRepository{
db: db,
}
}
func (repository PipRepository) CreatePip(pip price_feeds.PriceUpdate) error {
_, err := repository.db.Exec(`INSERT INTO maker.pips (block_number, header_id, usd_value) VALUES ($1, $2, $3::NUMERIC)`, pip.BlockNumber, pip.HeaderID, pip.UsdValue)
return err
}

51
pkg/transformers/price_feeds/pip/repository_test.go
View File

@ -1,51 +0,0 @@
package pip_test
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds"
"github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds/pip"
"github.com/vulcanize/vulcanizedb/test_config"
)
var _ = Describe("Pip repository", func() {
It("does not create a pip if no matching header", func() {
db := test_config.NewTestDB(core.Node{})
repository := pip.NewPipRepository(db)
priceUpdate := price_feeds.PriceUpdate{
BlockNumber: 0,
HeaderID: 0,
UsdValue: "123",
}
err := repository.CreatePip(priceUpdate)
Expect(err).To(HaveOccurred())
})
It("creates a pip when header exists", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
repository := pip.NewPipRepository(db)
headerRepository := repositories.NewHeaderRepository(db)
header := core.Header{BlockNumber: 12345}
headerID, err := headerRepository.CreateOrUpdateHeader(header)
Expect(err).NotTo(HaveOccurred())
priceUpdate := price_feeds.PriceUpdate{
BlockNumber: header.BlockNumber,
HeaderID: headerID,
UsdValue: "777.777",
}
err = repository.CreatePip(priceUpdate)
Expect(err).NotTo(HaveOccurred())
var dbPip price_feeds.PriceUpdate
err = db.Get(&dbPip, `SELECT block_number, header_id, usd_value FROM maker.pips WHERE block_number = $1`, header.BlockNumber)
Expect(err).NotTo(HaveOccurred())
Expect(dbPip).To(Equal(priceUpdate))
})
})

43
pkg/transformers/price_feeds/pip/transformer.go
View File

@ -1,43 +0,0 @@
package pip
import (
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds"
)
type PipTransformer struct {
fetcher IPipFetcher
repository IPipRepository
}
func NewPipTransformer(chain core.BlockChain, db *postgres.DB, contractAddress string) PipTransformer {
fetcher := NewPipFetcher(chain, contractAddress)
repository := NewPipRepository(db)
return PipTransformer{
fetcher: fetcher,
repository: repository,
}
}
func (transformer PipTransformer) Execute(header core.Header, headerID int64) error {
value, err := transformer.fetcher.FetchPipValue(header)
if err != nil {
if err == price_feeds.ErrNoMatchingLog {
return nil
}
return err
}
pip := getPip(value, header, headerID)
return transformer.repository.CreatePip(pip)
}
func getPip(logValue string, header core.Header, headerID int64) price_feeds.PriceUpdate {
valueInUSD := price_feeds.Convert("wad", logValue, 15)
pep := price_feeds.PriceUpdate{
BlockNumber: header.BlockNumber,
HeaderID: headerID,
UsdValue: valueInUSD,
}
return pep
}

45
pkg/transformers/price_feeds/pip/transformer_test.go
View File

@ -1,45 +0,0 @@
package pip_test
import (
"github.com/ethereum/go-ethereum/core/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds"
"github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds/pip"
"github.com/vulcanize/vulcanizedb/test_config"
)
var _ = Describe("Pip transformer", func() {
It("returns nil if no logs found", func() {
chain := fakes.NewMockBlockChain()
db := test_config.NewTestDB(core.Node{})
transformer := pip.NewPipTransformer(chain, db, "pip-contract-address")
err := transformer.Execute(core.Header{}, 123)
Expect(err).NotTo(HaveOccurred())
})
It("creates pip row for found log", func() {
chain := fakes.NewMockBlockChain()
chain.SetGetEthLogsWithCustomQueryReturnLogs([]types.Log{{Data: []byte{1, 2, 3, 4, 5}}})
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
header := core.Header{BlockNumber: 12345}
headerID, err := headerRepository.CreateOrUpdateHeader(header)
Expect(err).NotTo(HaveOccurred())
transformer := pip.NewPipTransformer(chain, db, "pip-contract-address")
err = transformer.Execute(header, headerID)
Expect(err).NotTo(HaveOccurred())
var dbPip price_feeds.PriceUpdate
err = db.Get(&dbPip, `SELECT block_number, header_id, usd_value FROM maker.pips WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(dbPip.BlockNumber).To(Equal(header.BlockNumber))
})
})

27
pkg/transformers/price_feeds/price_feeds_suite_test.go Normal file
View File

@ -0,0 +1,27 @@
// Copyright © 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package price_feeds_test
import (
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
func TestPriceFeeds(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "PriceFeeds Suite")
}

33
pkg/transformers/price_feeds/price_update.go
View File

@ -1,11 +1,34 @@
// Copyright © 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package price_feeds
import "math/big"
import (
"github.com/ethereum/go-ethereum/common"
"math/big"
)
type PriceUpdate struct {
BlockNumber int64 `db:"block_number"`
HeaderID int64 `db:"header_id"`
UsdValue string `db:"usd_value"`
type LogValueEntity struct {
Val common.Address
}
type PriceFeedModel struct {
BlockNumber uint64 `db:"block_number"`
HeaderID int64 `db:"header_id"`
MedianizerAddress []byte `db:"medianizer_address"`
UsdValue string `db:"usd_value"`
TransactionIndex uint `db:"tx_idx"`
}
func Convert(conversion string, value string, prec int) string {

51
pkg/transformers/price_feeds/rep/fetcher.go
View File

@ -1,51 +0,0 @@
package rep
import (
"github.com/ethereum/go-ethereum"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds"
"math/big"
)
type IRepFetcher interface {
FetchRepValue(header core.Header) (string, error)
}
type RepFetcher struct {
chain core.BlockChain
contractAddress string
}
func NewRepFetcher(chain core.BlockChain, contractAddress string) RepFetcher {
return RepFetcher{
chain: chain,
contractAddress: contractAddress,
}
}
func (fetcher RepFetcher) FetchRepValue(header core.Header) (string, error) {
blockNumber := big.NewInt(header.BlockNumber)
query := ethereum.FilterQuery{
FromBlock: blockNumber,
ToBlock: blockNumber,
Addresses: []common.Address{common.HexToAddress(fetcher.contractAddress)},
Topics: [][]common.Hash{{common.HexToHash(price_feeds.RepLogTopic0)}},
}
logs, err := fetcher.chain.GetEthLogsWithCustomQuery(query)
return fetcher.getLogValue(logs, err)
}
func (fetcher RepFetcher) getLogValue(logs []types.Log, err error) (string, error) {
if err != nil {
return "", err
}
if len(logs) < 1 {
return "", price_feeds.ErrNoMatchingLog
}
if len(logs) > 1 {
return "", price_feeds.ErrMultipleLogs
}
return string(logs[0].Data), nil
}

72
pkg/transformers/price_feeds/rep/fetcher_test.go
View File

@ -1,72 +0,0 @@
package rep_test
import (
"github.com/ethereum/go-ethereum"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds"
"github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds/rep"
"math/big"
)
var _ = Describe("Rep fetcher", func() {
It("gets logs describing updated rep/usd value", func() {
mockBlockChain := fakes.NewMockBlockChain()
mockBlockChain.SetGetEthLogsWithCustomQueryReturnLogs([]types.Log{{}})
contractAddress := "rep-contract-address"
fetcher := rep.NewRepFetcher(mockBlockChain, contractAddress)
blockNumber := int64(100)
header := core.Header{
BlockNumber: blockNumber,
Hash: "",
Raw: nil,
}
_, err := fetcher.FetchRepValue(header)
Expect(err).NotTo(HaveOccurred())
expectedQuery := ethereum.FilterQuery{
FromBlock: big.NewInt(blockNumber),
ToBlock: big.NewInt(blockNumber),
Addresses: []common.Address{common.HexToAddress(contractAddress)},
Topics: [][]common.Hash{{common.HexToHash(price_feeds.RepLogTopic0)}},
}
mockBlockChain.AssertGetEthLogsWithCustomQueryCalledWith(expectedQuery)
})
It("returns error if getting logs fails", func() {
mockBlockChain := fakes.NewMockBlockChain()
mockBlockChain.SetGetEthLogsWithCustomQueryErr(fakes.FakeError)
fetcher := rep.NewRepFetcher(mockBlockChain, "rep-contract-address")
_, err := fetcher.FetchRepValue(core.Header{})
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("returns no matching logs error if no logs returned", func() {
mockBlockChain := fakes.NewMockBlockChain()
fetcher := rep.NewRepFetcher(mockBlockChain, "rep-contract-address")
_, err := fetcher.FetchRepValue(core.Header{})
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(price_feeds.ErrNoMatchingLog))
})
It("returns error if more than one matching logs returned", func() {
mockBlockChain := fakes.NewMockBlockChain()
mockBlockChain.SetGetEthLogsWithCustomQueryReturnLogs([]types.Log{{}, {}})
fetcher := rep.NewRepFetcher(mockBlockChain, "rep-contract-address")
_, err := fetcher.FetchRepValue(core.Header{})
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(price_feeds.ErrMultipleLogs))
})
})

13
pkg/transformers/price_feeds/rep/rep_suite_test.go
View File

@ -1,13 +0,0 @@
package rep_test
import (
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
func TestRep(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Rep Suite")
}

25
pkg/transformers/price_feeds/rep/repository.go
View File

@ -1,25 +0,0 @@
package rep
import (
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds"
)
type IRepRepository interface {
CreateRep(rep price_feeds.PriceUpdate) error
}
type RepRepository struct {
db *postgres.DB
}
func NewRepRepository(db *postgres.DB) RepRepository {
return RepRepository{
db: db,
}
}
func (repository RepRepository) CreateRep(rep price_feeds.PriceUpdate) error {
_, err := repository.db.Exec(`INSERT INTO maker.reps (block_number, header_id, usd_value) VALUES ($1, $2, $3::NUMERIC)`, rep.BlockNumber, rep.HeaderID, rep.UsdValue)
return err
}

50
pkg/transformers/price_feeds/rep/repository_test.go
View File

@ -1,50 +0,0 @@
package rep_test
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds"
"github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds/rep"
"github.com/vulcanize/vulcanizedb/test_config"
)
var _ = Describe("Rep repository", func() {
It("returns header if matching header does not exist", func() {
db := test_config.NewTestDB(core.Node{})
repository := rep.NewRepRepository(db)
pepToAdd := price_feeds.PriceUpdate{
BlockNumber: 0,
HeaderID: 0,
UsdValue: "123.456",
}
err := repository.CreateRep(pepToAdd)
Expect(err).To(HaveOccurred())
})
It("creates a rep when matching header exists", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
repository := rep.NewRepRepository(db)
header := core.Header{BlockNumber: 12345}
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(header)
Expect(err).NotTo(HaveOccurred())
pepToAdd := price_feeds.PriceUpdate{
BlockNumber: header.BlockNumber,
HeaderID: headerID,
UsdValue: "123.456",
}
err = repository.CreateRep(pepToAdd)
Expect(err).NotTo(HaveOccurred())
var dbRep price_feeds.PriceUpdate
err = db.Get(&dbRep, `SELECT block_number, header_id, usd_value FROM maker.reps WHERE header_id = $1`, pepToAdd.HeaderID)
Expect(err).NotTo(HaveOccurred())
Expect(dbRep).To(Equal(pepToAdd))
})
})

43
pkg/transformers/price_feeds/rep/transformer.go
View File

@ -1,43 +0,0 @@
package rep
import (
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds"
)
type RepTransformer struct {
fetcher IRepFetcher
repository IRepRepository
}
func NewRepTransformer(chain core.BlockChain, db *postgres.DB, contractAddress string) RepTransformer {
fetcher := NewRepFetcher(chain, contractAddress)
repository := NewRepRepository(db)
return RepTransformer{
fetcher: fetcher,
repository: repository,
}
}
func (transformer RepTransformer) Execute(header core.Header, headerID int64) error {
logValue, err := transformer.fetcher.FetchRepValue(header)
if err != nil {
if err == price_feeds.ErrNoMatchingLog {
return nil
}
return err
}
rep := getRep(logValue, header, headerID)
return transformer.repository.CreateRep(rep)
}
func getRep(logValue string, header core.Header, headerID int64) price_feeds.PriceUpdate {
valueInUSD := price_feeds.Convert("wad", logValue, 15)
rep := price_feeds.PriceUpdate{
BlockNumber: header.BlockNumber,
HeaderID: headerID,
UsdValue: valueInUSD,
}
return rep
}

46
pkg/transformers/price_feeds/rep/transformer_test.go
View File

@ -1,46 +0,0 @@
package rep_test
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds"
"github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds/rep"
"github.com/vulcanize/vulcanizedb/test_config"
)
var _ = Describe("Rep transformer", func() {
It("returns nil if no logs found", func() {
chain := fakes.NewMockBlockChain()
db := test_config.NewTestDB(core.Node{})
transformer := rep.NewRepTransformer(chain, db, "rep-contract-address")
err := transformer.Execute(core.Header{}, 123)
Expect(err).NotTo(HaveOccurred())
})
It("creates rep row for found log", func() {
chain := fakes.NewMockBlockChain()
chain.SetGetEthLogsWithCustomQueryReturnLogs([]types.Log{{Data: []byte{1, 2, 3, 4, 5}}})
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
header := core.Header{BlockNumber: 12345}
headerID, err := headerRepository.CreateOrUpdateHeader(header)
Expect(err).NotTo(HaveOccurred())
transformer := rep.NewRepTransformer(chain, db, "rep-contract-address")
err = transformer.Execute(header, headerID)
Expect(err).NotTo(HaveOccurred())
var dbRep price_feeds.PriceUpdate
err = db.Get(&dbRep, `SELECT block_number, header_id, usd_value FROM maker.reps WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(dbRep.BlockNumber).To(Equal(header.BlockNumber))
})
})

56
pkg/transformers/price_feeds/repository.go Normal file
View File

@ -0,0 +1,56 @@
// Copyright © 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package price_feeds
import (
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
type IPriceFeedRepository interface {
Create(model PriceFeedModel) error
MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
}
type PriceFeedRepository struct {
db *postgres.DB
}
func NewPriceFeedRepository(db *postgres.DB) PriceFeedRepository {
return PriceFeedRepository{db: db}
}
func (repository PriceFeedRepository) Create(model PriceFeedModel) error {
_, err := repository.db.Exec(`INSERT INTO maker.price_feeds (block_number, header_id, medianizer_address, usd_value, tx_idx)
VALUES ($1, $2, $3, $4::NUMERIC, $5)`, model.BlockNumber, model.HeaderID, model.MedianizerAddress, model.UsdValue, model.TransactionIndex)
return err
}
func (repository PriceFeedRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
var result []core.Header
err := repository.db.Select(
&result,
`SELECT headers.id, headers.block_number FROM headers
LEFT JOIN maker.price_feeds on headers.id = header_id
WHERE header_id ISNULL
AND headers.block_number >= $1
AND headers.block_number <= $2
AND headers.eth_node_fingerprint = $3`,
startingBlockNumber,
endingBlockNumber,
repository.db.Node.ID,
)
return result, err
}

147
pkg/transformers/price_feeds/repository_test.go Normal file
View File

@ -0,0 +1,147 @@
// Copyright © 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package price_feeds_test
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds"
"github.com/vulcanize/vulcanizedb/test_config"
)
var _ = Describe("Price feeds repository", func() {
Describe("Create", func() {
It("persists a price feed update", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
blockNumber := uint64(12345)
header := core.Header{BlockNumber: int64(blockNumber)}
headerID, err := headerRepository.CreateOrUpdateHeader(header)
Expect(err).NotTo(HaveOccurred())
priceFeedUpdate := price_feeds.PriceFeedModel{
BlockNumber: blockNumber,
HeaderID: headerID,
MedianizerAddress: []byte{1, 2, 3, 4, 5},
UsdValue: "123.45",
TransactionIndex: 1,
}
priceFeedRepository := price_feeds.NewPriceFeedRepository(db)
err = priceFeedRepository.Create(priceFeedUpdate)
Expect(err).NotTo(HaveOccurred())
var dbPriceFeedUpdate price_feeds.PriceFeedModel
err = db.Get(&dbPriceFeedUpdate, `SELECT block_number, header_id, medianizer_address, usd_value, tx_idx FROM maker.price_feeds WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(dbPriceFeedUpdate).To(Equal(priceFeedUpdate))
})
It("does not duplicate price feed updates", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
blockNumber := uint64(12345)
header := core.Header{BlockNumber: int64(blockNumber)}
headerID, err := headerRepository.CreateOrUpdateHeader(header)
Expect(err).NotTo(HaveOccurred())
priceFeedUpdate := price_feeds.PriceFeedModel{
BlockNumber: blockNumber,
HeaderID: headerID,
MedianizerAddress: []byte{1, 2, 3, 4, 5},
UsdValue: "123.45",
TransactionIndex: 1,
}
priceFeedRepository := price_feeds.NewPriceFeedRepository(db)
err = priceFeedRepository.Create(priceFeedUpdate)
Expect(err).NotTo(HaveOccurred())
err = priceFeedRepository.Create(priceFeedUpdate)
Expect(err).To(HaveOccurred())
})
})
Describe("MissingHeaders", func() {
It("returns headers with no associated price feed event", func() {
node := core.Node{}
db := test_config.NewTestDB(node)
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
startingBlockNumber := int64(1)
priceFeedBlockNumber := int64(2)
endingBlockNumber := int64(3)
blockNumbers := []int64{startingBlockNumber, priceFeedBlockNumber, endingBlockNumber, endingBlockNumber + 1}
var headerIDs []int64
for _, n := range blockNumbers {
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
headerIDs = append(headerIDs, headerID)
Expect(err).NotTo(HaveOccurred())
}
priceFeedRepository := price_feeds.NewPriceFeedRepository(db)
priceFeedUpdate := price_feeds.PriceFeedModel{
BlockNumber: uint64(blockNumbers[1]),
HeaderID: headerIDs[1],
UsdValue: "123.45",
}
err := priceFeedRepository.Create(priceFeedUpdate)
Expect(err).NotTo(HaveOccurred())
headers, err := priceFeedRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
Expect(err).NotTo(HaveOccurred())
Expect(len(headers)).To(Equal(2))
Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
})
It("only returns headers associated with the current node", func() {
nodeOne := core.Node{}
db := test_config.NewTestDB(nodeOne)
test_config.CleanTestDB(db)
blockNumbers := []int64{1, 2, 3}
headerRepository := repositories.NewHeaderRepository(db)
nodeTwo := core.Node{ID: "second"}
dbTwo := test_config.NewTestDB(nodeTwo)
headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
var headerIDs []int64
for _, n := range blockNumbers {
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
Expect(err).NotTo(HaveOccurred())
headerIDs = append(headerIDs, headerID)
_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n})
Expect(err).NotTo(HaveOccurred())
}
priceFeedRepository := price_feeds.NewPriceFeedRepository(db)
priceFeedRepositoryTwo := price_feeds.NewPriceFeedRepository(dbTwo)
err := priceFeedRepository.Create(price_feeds.PriceFeedModel{
HeaderID: headerIDs[0],
UsdValue: "123.45",
})
Expect(err).NotTo(HaveOccurred())
nodeOneMissingHeaders, err := priceFeedRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])
Expect(err).NotTo(HaveOccurred())
Expect(len(nodeOneMissingHeaders)).To(Equal(len(blockNumbers) - 1))
nodeTwoMissingHeaders, err := priceFeedRepositoryTwo.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])
Expect(err).NotTo(HaveOccurred())
Expect(len(nodeTwoMissingHeaders)).To(Equal(len(blockNumbers)))
})
})
})

65
pkg/transformers/price_feeds/transformer.go Normal file
View File

@ -0,0 +1,65 @@
// Copyright © 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package price_feeds
import (
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
)
type PriceFeedTransformerInitializer struct {
Config IPriceFeedConfig
}
func (initializer PriceFeedTransformerInitializer) NewPriceFeedTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer {
converter := PriceFeedConverter{}
fetcher := NewPriceFeedFetcher(blockChain, initializer.Config.ContractAddresses)
repository := NewPriceFeedRepository(db)
return PriceFeedTransformer{
Config: initializer.Config,
converter: converter,
Fetcher: fetcher,
Repository: repository,
}
}
type PriceFeedTransformer struct {
Config IPriceFeedConfig
converter PriceFeedConverter
Fetcher IPriceFeedFetcher
Repository IPriceFeedRepository
}
func (transformer PriceFeedTransformer) Execute() error {
headers, err := transformer.Repository.MissingHeaders(transformer.Config.StartingBlockNumber, transformer.Config.EndingBlockNumber)
if err != nil {
return err
}
for _, header := range headers {
logs, err := transformer.Fetcher.FetchLogValues(header.BlockNumber)
if err != nil {
return err
}
for _, log := range logs {
model := transformer.converter.ToModel(log, header.Id)
err = transformer.Repository.Create(model)
if err != nil {
return err
}
}
}
return nil
}

150
pkg/transformers/price_feeds/transformer_test.go Normal file
View File

@ -0,0 +1,150 @@
// Copyright © 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package price_feeds_test
import (
"fmt"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds"
price_feeds2 "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks/price_feeds"
"math/big"
)
var _ = Describe("Price feed transformer", func() {
It("gets missing headers for price feeds", func() {
mockRepository := &price_feeds2.MockPriceFeedRepository{}
transformer := price_feeds.PriceFeedTransformer{
Config: price_feeds.PriceFeedConfig,
Fetcher: &price_feeds2.MockPriceFeedFetcher{},
Repository: mockRepository,
}
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
mockRepository.AssertMissingHeadersCalledwith(price_feeds.PriceFeedConfig.StartingBlockNumber, price_feeds.PriceFeedConfig.EndingBlockNumber)
})
It("returns error is missing headers call returns err", func() {
mockRepository := &price_feeds2.MockPriceFeedRepository{}
mockRepository.SetMissingHeadersErr(fakes.FakeError)
transformer := price_feeds.PriceFeedTransformer{
Fetcher: &price_feeds2.MockPriceFeedFetcher{},
Repository: mockRepository,
}
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("fetches logs for missing headers", func() {
mockRepository := &price_feeds2.MockPriceFeedRepository{}
blockNumberOne := int64(1)
blockNumberTwo := int64(2)
mockRepository.SetMissingHeaders([]core.Header{{BlockNumber: blockNumberOne}, {BlockNumber: blockNumberTwo}})
mockFetcher := &price_feeds2.MockPriceFeedFetcher{}
transformer := price_feeds.PriceFeedTransformer{
Fetcher: mockFetcher,
Repository: mockRepository,
}
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
mockFetcher.AssertFetchLogValuesCalledWith([]int64{blockNumberOne, blockNumberTwo})
})
It("returns err if fetcher returns err", func() {
mockRepository := &price_feeds2.MockPriceFeedRepository{}
mockRepository.SetMissingHeaders([]core.Header{{BlockNumber: 1}})
mockFetcher := &price_feeds2.MockPriceFeedFetcher{}
mockFetcher.SetReturnErr(fakes.FakeError)
transformer := price_feeds.PriceFeedTransformer{
Fetcher: mockFetcher,
Repository: mockRepository,
}
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("persists model converted from log", func() {
mockRepository := &price_feeds2.MockPriceFeedRepository{}
headerID := int64(11111)
mockRepository.SetMissingHeaders([]core.Header{{BlockNumber: 1, Id: headerID}})
mockFetcher := &price_feeds2.MockPriceFeedFetcher{}
blockNumber := uint64(22222)
txIndex := uint(33333)
usdValue := int64(44444)
etherMultiplier, _ := price_feeds.Ether.Int64()
rawUsdValue := big.NewInt(0)
rawUsdValue = rawUsdValue.Mul(big.NewInt(usdValue), big.NewInt(etherMultiplier))
address := common.BytesToAddress([]byte{1, 2, 3, 4, 5})
fakeLog := types.Log{
Address: address,
Topics: nil,
Data: rawUsdValue.Bytes(),
BlockNumber: blockNumber,
TxHash: common.Hash{},
TxIndex: txIndex,
BlockHash: common.Hash{},
Index: 0,
Removed: false,
}
mockFetcher.SetReturnLogs([]types.Log{fakeLog})
transformer := price_feeds.PriceFeedTransformer{
Fetcher: mockFetcher,
Repository: mockRepository,
}
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
expectedModel := price_feeds.PriceFeedModel{
BlockNumber: blockNumber,
HeaderID: headerID,
MedianizerAddress: address.Bytes(),
UsdValue: fmt.Sprintf("%d", usdValue),
TransactionIndex: txIndex,
}
mockRepository.AssertCreateCalledWith(expectedModel)
})
It("returns error if creating price feed update returns error", func() {
mockRepository := &price_feeds2.MockPriceFeedRepository{}
mockRepository.SetMissingHeaders([]core.Header{{BlockNumber: 1, Id: 2}})
mockRepository.SetCreateErr(fakes.FakeError)
mockFetcher := &price_feeds2.MockPriceFeedFetcher{}
mockFetcher.SetReturnLogs([]types.Log{{}})
transformer := price_feeds.PriceFeedTransformer{
Fetcher: mockFetcher,
Repository: mockRepository,
}
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
})

8
pkg/transformers/shared/transformer.go
View File

@ -27,6 +27,14 @@ type Transformer interface {
type TransformerInitializer func(db *postgres.DB, blockChain core.BlockChain) Transformer
type TransformerConfig struct {
ContractAddresses string
ContractAbi string
Topics []string
StartingBlockNumber int64
EndingBlockNumber int64
}
func HexToInt64(byteString string) int64 {
value := common.HexToHash(byteString)
return value.Big().Int64()

29
pkg/transformers/test_data/mocks/price_feeds/fetcher.go Normal file
View File

@ -0,0 +1,29 @@
package price_feeds
import (
"github.com/ethereum/go-ethereum/core/types"
. "github.com/onsi/gomega"
)
type MockPriceFeedFetcher struct {
passedBlockNumbers []int64
returnErr error
returnLogs []types.Log
}
func (fetcher *MockPriceFeedFetcher) SetReturnErr(err error) {
fetcher.returnErr = err
}
func (fetcher *MockPriceFeedFetcher) SetReturnLogs(logs []types.Log) {
fetcher.returnLogs = logs
}
func (fetcher *MockPriceFeedFetcher) FetchLogValues(blockNumber int64) ([]types.Log, error) {
fetcher.passedBlockNumbers = append(fetcher.passedBlockNumbers, blockNumber)
return fetcher.returnLogs, fetcher.returnErr
}
func (fetcher *MockPriceFeedFetcher) AssertFetchLogValuesCalledWith(blockNumbers []int64) {
Expect(fetcher.passedBlockNumbers).To(Equal(blockNumbers))
}

49
pkg/transformers/test_data/mocks/price_feeds/repository.go Normal file
View File

@ -0,0 +1,49 @@
package price_feeds
import (
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds"
)
type MockPriceFeedRepository struct {
createErr error
missingHeaders []core.Header
missingHeadersErr error
passedEndingBlockNumber int64
passedModel price_feeds.PriceFeedModel
passedStartingBlockNumber int64
}
func (repository *MockPriceFeedRepository) SetCreateErr(err error) {
repository.createErr = err
}
func (repository *MockPriceFeedRepository) SetMissingHeadersErr(err error) {
repository.missingHeadersErr = err
}
func (repository *MockPriceFeedRepository) SetMissingHeaders(headers []core.Header) {
repository.missingHeaders = headers
}
func (repository *MockPriceFeedRepository) Create(model price_feeds.PriceFeedModel) error {
repository.passedModel = model
return repository.createErr
}
func (repository *MockPriceFeedRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
repository.passedStartingBlockNumber = startingBlockNumber
repository.passedEndingBlockNumber = endingBlockNumber
return repository.missingHeaders, repository.missingHeadersErr
}
func (repository *MockPriceFeedRepository) AssertCreateCalledWith(model price_feeds.PriceFeedModel) {
Expect(repository.passedModel).To(Equal(model))
}
func (repository *MockPriceFeedRepository) AssertMissingHeadersCalledwith(startingBlockNumber, endingBlockNumber int64) {
Expect(repository.passedStartingBlockNumber).To(Equal(startingBlockNumber))
Expect(repository.passedEndingBlockNumber).To(Equal(endingBlockNumber))
}

9
pkg/transformers/transformer.go
View File

@ -1,9 +0,0 @@
package transformers
import (
"github.com/vulcanize/vulcanizedb/pkg/core"
)
type Transformer interface {
Execute(header core.Header, headerID int64) error
}

4
pkg/transformers/transformers.go
View File

@ -17,6 +17,7 @@ package transformers
import (
"github.com/vulcanize/vulcanizedb/pkg/transformers/flip_kick"
"github.com/vulcanize/vulcanizedb/pkg/transformers/frob"
"github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
)
@ -25,8 +26,11 @@ func TransformerInitializers() []shared.TransformerInitializer {
flipKickTransformerInitializer := flip_kick.FlipKickTransformerInitializer{Config: flipKickConfig}
frobConfig := frob.FrobConfig
frobTransformerInitializer := frob.FrobTransformerInitializer{Config: frobConfig}
priceFeedConfig := price_feeds.PriceFeedConfig
priceFeedTransformerInitializer := price_feeds.PriceFeedTransformerInitializer{Config: priceFeedConfig}
return []shared.TransformerInitializer{
flipKickTransformerInitializer.NewFlipKickTransformer,
frobTransformerInitializer.NewFrobTransformer,
priceFeedTransformerInitializer.NewPriceFeedTransformer,
}
}

3
test_config/test_config.go
View File

@ -77,9 +77,6 @@ func CleanTestDB(db *postgres.DB) {
db.MustExec("DELETE FROM headers")
db.MustExec("DELETE FROM log_filters")
db.MustExec("DELETE FROM logs")
db.MustExec("DELETE FROM maker.peps")
db.MustExec("DELETE FROM maker.pips")
db.MustExec("DELETE FROM maker.reps")
db.MustExec("DELETE FROM receipts")
db.MustExec("DELETE FROM transactions")
db.MustExec("DELETE FROM watched_contracts")