From db4b8913988eebfbc9dc18c1102559c1861f1bc6 Mon Sep 17 00:00:00 2001 From: Rob Mulholand Date: Wed, 29 Aug 2018 09:12:29 -0500 Subject: [PATCH] Backfill Pit.file log events --- .../1535667935_create_pit_file_table.down.sql | 1 + .../1535667935_create_pit_file_table.up.sql | 10 ++ db/schema.sql | 66 +++++++ pkg/transformers/bite/bite_suite_test.go | 6 + pkg/transformers/bite/config.go | 2 +- pkg/transformers/bite/transformer.go | 4 +- pkg/transformers/bite/transfromer_test.go | 6 +- pkg/transformers/flip_kick/config.go | 2 +- pkg/transformers/flip_kick/transformer.go | 4 +- .../flip_kick/transformer_test.go | 6 +- pkg/transformers/frob/config.go | 2 +- pkg/transformers/frob/transformer.go | 4 +- pkg/transformers/frob/transformer_test.go | 4 +- pkg/transformers/pit_file/config.go | 13 ++ pkg/transformers/pit_file/converter.go | 32 ++++ pkg/transformers/pit_file/converter_test.go | 20 +++ pkg/transformers/pit_file/model.go | 9 + .../pit_file/pit_file_suite_test.go | 13 ++ pkg/transformers/pit_file/repository.go | 48 ++++++ pkg/transformers/pit_file/repository_test.go | 130 ++++++++++++++ pkg/transformers/pit_file/transformer.go | 56 ++++++ pkg/transformers/pit_file/transformer_test.go | 162 ++++++++++++++++++ pkg/transformers/shared/constants.go | 8 +- pkg/transformers/shared/transformer.go | 2 +- pkg/transformers/tend/config.go | 2 +- pkg/transformers/tend/transformer.go | 4 +- pkg/transformers/tend/transformer_test.go | 2 +- .../test_data/mocks/pit_file/converter.go | 26 +++ .../test_data/mocks/pit_file/repository.go | 42 +++++ pkg/transformers/test_data/pit_file.go | 41 +++++ pkg/transformers/transformers.go | 4 + test_config/test_config.go | 1 + 32 files changed, 707 insertions(+), 25 deletions(-) create mode 100644 db/migrations/1535667935_create_pit_file_table.down.sql create mode 100644 db/migrations/1535667935_create_pit_file_table.up.sql create mode 100644 pkg/transformers/pit_file/config.go create mode 100644 pkg/transformers/pit_file/converter.go create mode 100644 pkg/transformers/pit_file/converter_test.go create mode 100644 pkg/transformers/pit_file/model.go create mode 100644 pkg/transformers/pit_file/pit_file_suite_test.go create mode 100644 pkg/transformers/pit_file/repository.go create mode 100644 pkg/transformers/pit_file/repository_test.go create mode 100644 pkg/transformers/pit_file/transformer.go create mode 100644 pkg/transformers/pit_file/transformer_test.go create mode 100644 pkg/transformers/test_data/mocks/pit_file/converter.go create mode 100644 pkg/transformers/test_data/mocks/pit_file/repository.go create mode 100644 pkg/transformers/test_data/pit_file.go diff --git a/db/migrations/1535667935_create_pit_file_table.down.sql b/db/migrations/1535667935_create_pit_file_table.down.sql new file mode 100644 index 00000000..4f92d89f --- /dev/null +++ b/db/migrations/1535667935_create_pit_file_table.down.sql @@ -0,0 +1 @@ +DROP TABLE maker.pit_file; \ No newline at end of file diff --git a/db/migrations/1535667935_create_pit_file_table.up.sql b/db/migrations/1535667935_create_pit_file_table.up.sql new file mode 100644 index 00000000..ff0312b4 --- /dev/null +++ b/db/migrations/1535667935_create_pit_file_table.up.sql @@ -0,0 +1,10 @@ +CREATE TABLE maker.pit_file ( + id SERIAL PRIMARY KEY, + header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE, + ilk TEXT, + what TEXT, + risk NUMERIC, + tx_idx INTEGER NOT NUll, + raw_log JSONB, + UNIQUE (header_id, tx_idx) +); \ No newline at end of file diff --git a/db/schema.sql b/db/schema.sql index 761378b1..8cc66b1d 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -155,6 +155,41 @@ CREATE SEQUENCE maker.frob_id_seq ALTER SEQUENCE maker.frob_id_seq OWNED BY maker.frob.id; +-- +-- Name: pit_file; Type: TABLE; Schema: maker; Owner: - +-- + +CREATE TABLE maker.pit_file ( + id integer NOT NULL, + header_id integer NOT NULL, + ilk text, + what text, + risk numeric, + tx_idx integer NOT NULL, + raw_log jsonb +); + + +-- +-- Name: pit_file_id_seq; Type: SEQUENCE; Schema: maker; Owner: - +-- + +CREATE SEQUENCE maker.pit_file_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: pit_file_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: - +-- + +ALTER SEQUENCE maker.pit_file_id_seq OWNED BY maker.pit_file.id; + + -- -- Name: price_feeds; Type: TABLE; Schema: maker; Owner: - -- @@ -619,6 +654,13 @@ ALTER TABLE ONLY maker.flip_kick ALTER COLUMN db_id SET DEFAULT nextval('maker.f ALTER TABLE ONLY maker.frob ALTER COLUMN id SET DEFAULT nextval('maker.frob_id_seq'::regclass); +-- +-- Name: pit_file id; Type: DEFAULT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.pit_file ALTER COLUMN id SET DEFAULT nextval('maker.pit_file_id_seq'::regclass); + + -- -- Name: price_feeds id; Type: DEFAULT; Schema: maker; Owner: - -- @@ -744,6 +786,22 @@ ALTER TABLE ONLY maker.frob ADD CONSTRAINT frob_pkey PRIMARY KEY (id); +-- +-- Name: pit_file pit_file_header_id_tx_idx_key; Type: CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.pit_file + ADD CONSTRAINT pit_file_header_id_tx_idx_key UNIQUE (header_id, tx_idx); + + +-- +-- Name: pit_file pit_file_pkey; Type: CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.pit_file + ADD CONSTRAINT pit_file_pkey PRIMARY KEY (id); + + -- -- Name: price_feeds price_feeds_header_id_medianizer_address_tx_idx_key; Type: CONSTRAINT; Schema: maker; Owner: - -- @@ -939,6 +997,14 @@ ALTER TABLE ONLY maker.price_feeds ADD CONSTRAINT headers_fk FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE; +-- +-- Name: pit_file pit_file_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.pit_file + ADD CONSTRAINT pit_file_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE; + + -- -- Name: tend tend_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - -- diff --git a/pkg/transformers/bite/bite_suite_test.go b/pkg/transformers/bite/bite_suite_test.go index ab9920d3..e5a93b26 100644 --- a/pkg/transformers/bite/bite_suite_test.go +++ b/pkg/transformers/bite/bite_suite_test.go @@ -19,9 +19,15 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "io/ioutil" + "log" ) func TestBite(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "Bite Suite") } + +var _ = BeforeSuite(func() { + log.SetOutput(ioutil.Discard) +}) diff --git a/pkg/transformers/bite/config.go b/pkg/transformers/bite/config.go index 6c743cf5..f05a5c29 100644 --- a/pkg/transformers/bite/config.go +++ b/pkg/transformers/bite/config.go @@ -21,7 +21,7 @@ import ( ) var BiteConfig = shared.TransformerConfig{ - ContractAddresses: "0xe0f0fa6982c59d8aa4ae0134bfe048327bd788cacf758b643ca41f055ffce76c", //this is a temporary address deployed locally + ContractAddress: "0xe0f0fa6982c59d8aa4ae0134bfe048327bd788cacf758b643ca41f055ffce76c", //this is a temporary address deployed locally ContractAbi: BiteABI, Topics: []string{BiteSignature}, StartingBlockNumber: 0, diff --git a/pkg/transformers/bite/transformer.go b/pkg/transformers/bite/transformer.go index 3f90157d..7a68311a 100644 --- a/pkg/transformers/bite/transformer.go +++ b/pkg/transformers/bite/transformer.go @@ -59,14 +59,14 @@ func (b BiteTransformer) Execute() error { } for _, header := range missingHeaders { - ethLogs, err := b.Fetcher.FetchLogs(config.ContractAddresses, topics, header.BlockNumber) + ethLogs, err := b.Fetcher.FetchLogs(config.ContractAddress, topics, header.BlockNumber) if err != nil { log.Println("Error fetching matching logs:", err) return err } for _, ethLog := range ethLogs { - entity, err := b.Converter.ToEntity(config.ContractAddresses, config.ContractAbi, ethLog) + entity, err := b.Converter.ToEntity(config.ContractAddress, config.ContractAbi, ethLog) model, err := b.Converter.ToModel(entity) if err != nil { log.Println("Error converting logs:", err) diff --git a/pkg/transformers/bite/transfromer_test.go b/pkg/transformers/bite/transfromer_test.go index cdc6c8d9..21e43fbb 100644 --- a/pkg/transformers/bite/transfromer_test.go +++ b/pkg/transformers/bite/transfromer_test.go @@ -53,7 +53,7 @@ var _ = Describe("Bite Transformer", func() { } testConfig = shared.TransformerConfig{ - ContractAddresses: "0x12345", + ContractAddress: "0x12345", ContractAbi: "test abi", Topics: []string{shared.BiteSignature}, StartingBlockNumber: blockNumber1, @@ -85,7 +85,7 @@ var _ = Describe("Bite Transformer", func() { Expect(err).NotTo(HaveOccurred()) Expect(fetcher.FetchedBlocks).To(Equal([]int64{blockNumber1, blockNumber2})) Expect(fetcher.FetchedTopics).To(Equal(expectedTopics)) - Expect(fetcher.FetchedContractAddress).To(Equal(bite.BiteConfig.ContractAddresses)) + Expect(fetcher.FetchedContractAddress).To(Equal(bite.BiteConfig.ContractAddress)) }) It("returns an error if fetching logs fails", func() { @@ -103,7 +103,7 @@ var _ = Describe("Bite Transformer", func() { err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(converter.ConverterContract).To(Equal(bite.BiteConfig.ContractAddresses)) + Expect(converter.ConverterContract).To(Equal(bite.BiteConfig.ContractAddress)) Expect(converter.ConverterAbi).To(Equal(bite.BiteConfig.ContractAbi)) Expect(converter.LogsToConvert).To(Equal([]types.Log{test_data.EthBiteLog})) }) diff --git a/pkg/transformers/flip_kick/config.go b/pkg/transformers/flip_kick/config.go index 5a34fa93..c5e1448f 100644 --- a/pkg/transformers/flip_kick/config.go +++ b/pkg/transformers/flip_kick/config.go @@ -17,7 +17,7 @@ package flip_kick import "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" var FlipKickConfig = shared.TransformerConfig{ - ContractAddresses: shared.FlipperContractAddress, + ContractAddress: shared.FlipperContractAddress, ContractAbi: shared.FlipperABI, Topics: []string{shared.FlipKickSignature}, StartingBlockNumber: 0, diff --git a/pkg/transformers/flip_kick/transformer.go b/pkg/transformers/flip_kick/transformer.go index 0f700b57..a7331073 100644 --- a/pkg/transformers/flip_kick/transformer.go +++ b/pkg/transformers/flip_kick/transformer.go @@ -91,13 +91,13 @@ func (fkt FlipKickTransformer) Execute() error { log.Printf("Fetching event logs for %d headers \n", len(headers)) var resultingErrors []error for _, header := range headers { - ethLogs, err := fkt.Fetcher.FetchLogs(config.ContractAddresses, topics, header.BlockNumber) + ethLogs, err := fkt.Fetcher.FetchLogs(config.ContractAddress, topics, header.BlockNumber) if err != nil { resultingErrors = append(resultingErrors, newTransformerError(err, header.BlockNumber, FetcherError)) } for _, ethLog := range ethLogs { - entity, err := fkt.Converter.ToEntity(config.ContractAddresses, config.ContractAbi, ethLog) + entity, err := fkt.Converter.ToEntity(config.ContractAddress, config.ContractAbi, ethLog) if err != nil { resultingErrors = append(resultingErrors, newTransformerError(err, header.BlockNumber, LogToEntityError)) } diff --git a/pkg/transformers/flip_kick/transformer_test.go b/pkg/transformers/flip_kick/transformer_test.go index 0f14665a..7e4350df 100644 --- a/pkg/transformers/flip_kick/transformer_test.go +++ b/pkg/transformers/flip_kick/transformer_test.go @@ -54,7 +54,7 @@ var _ = Describe("FlipKick Transformer", func() { startingBlockNumber := rand.Int63() testConfig = shared.TransformerConfig{ - ContractAddresses: "0x12345", + ContractAddress: "0x12345", ContractAbi: "test abi", Topics: []string{shared.FlipKickSignature}, StartingBlockNumber: startingBlockNumber, @@ -83,7 +83,7 @@ var _ = Describe("FlipKick Transformer", func() { err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(fetcher.FetchedContractAddress).To(Equal(testConfig.ContractAddresses)) + Expect(fetcher.FetchedContractAddress).To(Equal(testConfig.ContractAddress)) Expect(fetcher.FetchedTopics).To(Equal(expectedTopics)) Expect(fetcher.FetchedBlocks).To(Equal([]int64{blockNumber})) }) @@ -100,7 +100,7 @@ var _ = Describe("FlipKick Transformer", func() { err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(converter.ConverterContract).To(Equal(testConfig.ContractAddresses)) + Expect(converter.ConverterContract).To(Equal(testConfig.ContractAddress)) Expect(converter.ConverterAbi).To(Equal(testConfig.ContractAbi)) Expect(converter.LogsToConvert).To(Equal(logs)) Expect(converter.EntitiesToConvert).To(Equal([]flip_kick.FlipKickEntity{test_data.FlipKickEntity})) diff --git a/pkg/transformers/frob/config.go b/pkg/transformers/frob/config.go index cd824f06..55d209fe 100644 --- a/pkg/transformers/frob/config.go +++ b/pkg/transformers/frob/config.go @@ -17,7 +17,7 @@ package frob import "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" var FrobConfig = shared.TransformerConfig{ - ContractAddresses: "0xff3f2400f1600f3f493a9a92704a29b96795af1a", //this is a temporary address deployed locally + ContractAddress: "0xff3f2400f1600f3f493a9a92704a29b96795af1a", //this is a temporary address deployed locally ContractAbi: FrobABI, Topics: []string{FrobEventSignature}, StartingBlockNumber: 0, diff --git a/pkg/transformers/frob/transformer.go b/pkg/transformers/frob/transformer.go index f72d2b08..324e345e 100644 --- a/pkg/transformers/frob/transformer.go +++ b/pkg/transformers/frob/transformer.go @@ -52,12 +52,12 @@ func (transformer FrobTransformer) Execute() error { } for _, header := range missingHeaders { topics := [][]common.Hash{{common.HexToHash(FrobEventSignature)}} - matchingLogs, err := transformer.Fetcher.FetchLogs(FrobConfig.ContractAddresses, topics, header.BlockNumber) + matchingLogs, err := transformer.Fetcher.FetchLogs(FrobConfig.ContractAddress, topics, header.BlockNumber) if err != nil { return err } for _, log := range matchingLogs { - entity, err := transformer.Converter.ToEntity(FrobConfig.ContractAddresses, FrobConfig.ContractAbi, log) + entity, err := transformer.Converter.ToEntity(FrobConfig.ContractAddress, FrobConfig.ContractAbi, log) if err != nil { return err } diff --git a/pkg/transformers/frob/transformer_test.go b/pkg/transformers/frob/transformer_test.go index ca7dfe31..36e7fa4c 100644 --- a/pkg/transformers/frob/transformer_test.go +++ b/pkg/transformers/frob/transformer_test.go @@ -74,7 +74,7 @@ var _ = Describe("Frob transformer", func() { Expect(err).NotTo(HaveOccurred()) Expect(fetcher.FetchedBlocks).To(Equal([]int64{1, 2})) - Expect(fetcher.FetchedContractAddress).To(Equal(frob.FrobConfig.ContractAddresses)) + Expect(fetcher.FetchedContractAddress).To(Equal(frob.FrobConfig.ContractAddress)) Expect(fetcher.FetchedTopics).To(Equal([][]common.Hash{{common.HexToHash(frob.FrobEventSignature)}})) }) @@ -110,7 +110,7 @@ var _ = Describe("Frob transformer", func() { err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(converter.PassedContractAddress).To(Equal(frob.FrobConfig.ContractAddresses)) + Expect(converter.PassedContractAddress).To(Equal(frob.FrobConfig.ContractAddress)) Expect(converter.PassedContractABI).To(Equal(frob.FrobConfig.ContractAbi)) Expect(converter.PassedLog).To(Equal(test_data.EthFrobLog)) Expect(converter.PassedEntity).To(Equal(test_data.FrobEntity)) diff --git a/pkg/transformers/pit_file/config.go b/pkg/transformers/pit_file/config.go new file mode 100644 index 00000000..d7b9516c --- /dev/null +++ b/pkg/transformers/pit_file/config.go @@ -0,0 +1,13 @@ +package pit_file + +import ( + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" +) + +var PitFileConfig = shared.TransformerConfig{ + ContractAddress: "0xff3f2400f1600f3f493a9a92704a29b96795af1a", // temporary address from Ganache deploy + ContractAbi: shared.PitABI, + Topics: []string{shared.PitFileSignature}, + StartingBlockNumber: 0, + EndingBlockNumber: 100, +} diff --git a/pkg/transformers/pit_file/converter.go b/pkg/transformers/pit_file/converter.go new file mode 100644 index 00000000..0bcc4116 --- /dev/null +++ b/pkg/transformers/pit_file/converter.go @@ -0,0 +1,32 @@ +package pit_file + +import ( + "bytes" + "encoding/json" + "github.com/ethereum/go-ethereum/core/types" + "math/big" +) + +type Converter interface { + ToModel(contractAddress string, contractAbi string, ethLog types.Log) (PitFileModel, error) +} + +type PitFileConverter struct { +} + +func (PitFileConverter) ToModel(contractAddress string, contractAbi string, ethLog types.Log) (entity PitFileModel, err error) { + ilk := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00")) + what := string(bytes.Trim(ethLog.Topics[3].Bytes(), "\x00")) + itemByteLength := 32 + riskBytes := ethLog.Data[len(ethLog.Data)-itemByteLength:] + risk := big.NewInt(0).SetBytes(riskBytes).String() + + raw, err := json.Marshal(ethLog) + return PitFileModel{ + Ilk: ilk, + What: what, + Risk: risk, + TransactionIndex: ethLog.TxIndex, + Raw: raw, + }, err +} diff --git a/pkg/transformers/pit_file/converter_test.go b/pkg/transformers/pit_file/converter_test.go new file mode 100644 index 00000000..30e97fd6 --- /dev/null +++ b/pkg/transformers/pit_file/converter_test.go @@ -0,0 +1,20 @@ +package pit_file_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" +) + +var _ = Describe("Pit file converter", func() { + It("converts a log to an entity", func() { + converter := pit_file.PitFileConverter{} + + model, err := converter.ToModel(test_data.PitAddress, shared.PitABI, test_data.EthPitFileLog) + + Expect(err).NotTo(HaveOccurred()) + Expect(model).To(Equal(test_data.PitFileModel)) + }) +}) diff --git a/pkg/transformers/pit_file/model.go b/pkg/transformers/pit_file/model.go new file mode 100644 index 00000000..d42ba67d --- /dev/null +++ b/pkg/transformers/pit_file/model.go @@ -0,0 +1,9 @@ +package pit_file + +type PitFileModel struct { + Ilk string + What string + Risk string + TransactionIndex uint `db:"tx_idx"` + Raw []byte `db:"raw_log"` +} diff --git a/pkg/transformers/pit_file/pit_file_suite_test.go b/pkg/transformers/pit_file/pit_file_suite_test.go new file mode 100644 index 00000000..c992d4c9 --- /dev/null +++ b/pkg/transformers/pit_file/pit_file_suite_test.go @@ -0,0 +1,13 @@ +package pit_file_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestPitFile(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "PitFile Suite") +} diff --git a/pkg/transformers/pit_file/repository.go b/pkg/transformers/pit_file/repository.go new file mode 100644 index 00000000..4184269a --- /dev/null +++ b/pkg/transformers/pit_file/repository.go @@ -0,0 +1,48 @@ +package pit_file + +import ( + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" +) + +type Repository interface { + Create(headerID int64, transactionIndex uint, model PitFileModel) error + MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) +} + +type PitFileRepository struct { + db *postgres.DB +} + +func NewPitFileRepository(db *postgres.DB) PitFileRepository { + return PitFileRepository{ + db: db, + } +} + +func (repository PitFileRepository) Create(headerID int64, transactionIndex uint, model PitFileModel) error { + _, err := repository.db.Exec( + `INSERT into maker.pit_file (header_id, ilk, what, risk, tx_idx, raw_log) + VALUES($1, $2, $3, $4::NUMERIC, $5, $6)`, + headerID, model.Ilk, model.What, model.Risk, model.TransactionIndex, model.Raw, + ) + return err +} + +func (repository PitFileRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + var result []core.Header + err := repository.db.Select( + &result, + `SELECT headers.id, headers.block_number FROM headers + LEFT JOIN maker.pit_file on headers.id = header_id + WHERE header_id ISNULL + AND headers.block_number >= $1 + AND headers.block_number <= $2 + AND headers.eth_node_fingerprint = $3`, + startingBlockNumber, + endingBlockNumber, + repository.db.Node.ID, + ) + + return result, err +} diff --git a/pkg/transformers/pit_file/repository_test.go b/pkg/transformers/pit_file/repository_test.go new file mode 100644 index 00000000..d86a46e9 --- /dev/null +++ b/pkg/transformers/pit_file/repository_test.go @@ -0,0 +1,130 @@ +package pit_file_test + +import ( + "database/sql" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file" + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" + "github.com/vulcanize/vulcanizedb/test_config" +) + +var _ = Describe("Pit file repository", func() { + Describe("Create", func() { + It("adds a pit file event", func() { + db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{}) + Expect(err).NotTo(HaveOccurred()) + pitFileRepository := pit_file.NewPitFileRepository(db) + + err = pitFileRepository.Create(headerID, 123, test_data.PitFileModel) + + Expect(err).NotTo(HaveOccurred()) + var dbPitFile pit_file.PitFileModel + err = db.Get(&dbPitFile, `SELECT ilk, what, risk, tx_idx, raw_log FROM maker.pit_file WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(dbPitFile.Ilk).To(Equal(test_data.PitFileModel.Ilk)) + Expect(dbPitFile.What).To(Equal(test_data.PitFileModel.What)) + Expect(dbPitFile.Risk).To(Equal(test_data.PitFileModel.Risk)) + Expect(dbPitFile.TransactionIndex).To(Equal(test_data.PitFileModel.TransactionIndex)) + // The raw bytes scanned out of the DB are compressed (missing whitespace), such that they don't strictly equal the input bytes + Expect(dbPitFile.Raw).To(MatchJSON(test_data.PitFileModel.Raw)) + }) + + It("does not duplicate pit file events", func() { + db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{}) + Expect(err).NotTo(HaveOccurred()) + pitFileRepository := pit_file.NewPitFileRepository(db) + err = pitFileRepository.Create(headerID, 123, test_data.PitFileModel) + Expect(err).NotTo(HaveOccurred()) + + err = pitFileRepository.Create(headerID, 123, test_data.PitFileModel) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint")) + }) + + It("removes pit file if corresponding header is deleted", func() { + db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{}) + Expect(err).NotTo(HaveOccurred()) + pitFileRepository := pit_file.NewPitFileRepository(db) + err = pitFileRepository.Create(headerID, 123, test_data.PitFileModel) + Expect(err).NotTo(HaveOccurred()) + + _, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID) + + Expect(err).NotTo(HaveOccurred()) + var dbPitFile pit_file.PitFileModel + err = db.Get(&dbPitFile, `SELECT ilk, what, risk, tx_idx, raw_log FROM maker.pit_file WHERE header_id = $1`, headerID) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(sql.ErrNoRows)) + }) + }) + + Describe("MissingHeaders", func() { + It("returns headers with no associated pit file event", func() { + db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + startingBlockNumber := int64(1) + pitFileBlockNumber := int64(2) + endingBlockNumber := int64(3) + blockNumbers := []int64{startingBlockNumber, pitFileBlockNumber, endingBlockNumber, endingBlockNumber + 1} + var headerIDs []int64 + for _, n := range blockNumbers { + headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + headerIDs = append(headerIDs, headerID) + Expect(err).NotTo(HaveOccurred()) + } + pitFileRepository := pit_file.NewPitFileRepository(db) + err := pitFileRepository.Create(headerIDs[1], 123, test_data.PitFileModel) + Expect(err).NotTo(HaveOccurred()) + + headers, err := pitFileRepository.MissingHeaders(startingBlockNumber, endingBlockNumber) + + Expect(err).NotTo(HaveOccurred()) + Expect(len(headers)).To(Equal(2)) + Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber))) + Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber))) + }) + + It("only returns headers associated with the current node", func() { + db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + blockNumbers := []int64{1, 2, 3} + headerRepository := repositories.NewHeaderRepository(db) + dbTwo := test_config.NewTestDB(core.Node{ID: "second"}) + headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo) + var headerIDs []int64 + for _, n := range blockNumbers { + headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + Expect(err).NotTo(HaveOccurred()) + headerIDs = append(headerIDs, headerID) + _, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + Expect(err).NotTo(HaveOccurred()) + } + pitFileRepository := pit_file.NewPitFileRepository(db) + pitFileRepositoryTwo := pit_file.NewPitFileRepository(dbTwo) + err := pitFileRepository.Create(headerIDs[0], 0, test_data.PitFileModel) + Expect(err).NotTo(HaveOccurred()) + + nodeOneMissingHeaders, err := pitFileRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) + Expect(err).NotTo(HaveOccurred()) + Expect(len(nodeOneMissingHeaders)).To(Equal(len(blockNumbers) - 1)) + + nodeTwoMissingHeaders, err := pitFileRepositoryTwo.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) + Expect(err).NotTo(HaveOccurred()) + Expect(len(nodeTwoMissingHeaders)).To(Equal(len(blockNumbers))) + }) + }) +}) diff --git a/pkg/transformers/pit_file/transformer.go b/pkg/transformers/pit_file/transformer.go new file mode 100644 index 00000000..c6c38ad3 --- /dev/null +++ b/pkg/transformers/pit_file/transformer.go @@ -0,0 +1,56 @@ +package pit_file + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" +) + +type PitFileTransformer struct { + Config shared.TransformerConfig + Converter Converter + Fetcher shared.LogFetcher + Repository Repository +} + +type PitFileTransformerInitializer struct { + Config shared.TransformerConfig +} + +func (initializer PitFileTransformerInitializer) NewPitFileTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer { + converter := PitFileConverter{} + fetcher := shared.NewFetcher(blockChain) + repository := NewPitFileRepository(db) + return PitFileTransformer{ + Config: initializer.Config, + Converter: converter, + Fetcher: fetcher, + Repository: repository, + } +} + +func (transformer PitFileTransformer) Execute() error { + missingHeaders, err := transformer.Repository.MissingHeaders(transformer.Config.StartingBlockNumber, transformer.Config.EndingBlockNumber) + if err != nil { + return err + } + for _, header := range missingHeaders { + topics := [][]common.Hash{{common.HexToHash(shared.PitFileSignature)}} + matchingLogs, err := transformer.Fetcher.FetchLogs(PitFileConfig.ContractAddress, topics, header.BlockNumber) + if err != nil { + return err + } + for _, log := range matchingLogs { + model, err := transformer.Converter.ToModel(PitFileConfig.ContractAddress, shared.PitABI, log) + if err != nil { + return err + } + err = transformer.Repository.Create(header.Id, log.TxIndex, model) + if err != nil { + return err + } + } + } + return nil +} diff --git a/pkg/transformers/pit_file/transformer_test.go b/pkg/transformers/pit_file/transformer_test.go new file mode 100644 index 00000000..e9351b3c --- /dev/null +++ b/pkg/transformers/pit_file/transformer_test.go @@ -0,0 +1,162 @@ +package pit_file_test + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/fakes" + "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks" + pit_file_mocks "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks/pit_file" +) + +var _ = Describe("Pit file transformer", func() { + It("gets missing headers for block numbers specified in config", func() { + repository := &pit_file_mocks.MockPitFileRepository{} + transformer := pit_file.PitFileTransformer{ + Config: pit_file.PitFileConfig, + Fetcher: &mocks.MockLogFetcher{}, + Converter: &pit_file_mocks.MockPitFileConverter{}, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + Expect(repository.PassedStartingBlockNumber).To(Equal(pit_file.PitFileConfig.StartingBlockNumber)) + Expect(repository.PassedEndingBlockNumber).To(Equal(pit_file.PitFileConfig.EndingBlockNumber)) + }) + + It("returns error if repository returns error for missing headers", func() { + repository := &pit_file_mocks.MockPitFileRepository{} + repository.SetMissingHeadersErr(fakes.FakeError) + transformer := pit_file.PitFileTransformer{ + Fetcher: &mocks.MockLogFetcher{}, + Converter: &pit_file_mocks.MockPitFileConverter{}, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + + It("fetches logs for missing headers", func() { + fetcher := &mocks.MockLogFetcher{} + repository := &pit_file_mocks.MockPitFileRepository{} + repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}, {BlockNumber: 2}}) + transformer := pit_file.PitFileTransformer{ + Fetcher: fetcher, + Converter: &pit_file_mocks.MockPitFileConverter{}, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + Expect(fetcher.FetchedBlocks).To(Equal([]int64{1, 2})) + Expect(fetcher.FetchedContractAddress).To(Equal(pit_file.PitFileConfig.ContractAddress)) + Expect(fetcher.FetchedTopics).To(Equal([][]common.Hash{{common.HexToHash(shared.PitFileSignature)}})) + }) + + It("returns error if fetcher returns error", func() { + fetcher := &mocks.MockLogFetcher{} + fetcher.SetFetcherError(fakes.FakeError) + repository := &pit_file_mocks.MockPitFileRepository{} + repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) + transformer := pit_file.PitFileTransformer{ + Fetcher: fetcher, + Converter: &pit_file_mocks.MockPitFileConverter{}, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + + It("converts matching logs", func() { + converter := &pit_file_mocks.MockPitFileConverter{} + fetcher := &mocks.MockLogFetcher{} + fetcher.SetFetchedLogs([]types.Log{test_data.EthPitFileLog}) + repository := &pit_file_mocks.MockPitFileRepository{} + repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) + transformer := pit_file.PitFileTransformer{ + Fetcher: fetcher, + Converter: converter, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + Expect(converter.PassedContractAddress).To(Equal(pit_file.PitFileConfig.ContractAddress)) + Expect(converter.PassedContractABI).To(Equal(pit_file.PitFileConfig.ContractAbi)) + Expect(converter.PassedLog).To(Equal(test_data.EthPitFileLog)) + }) + + It("returns error if converter returns error", func() { + converter := &pit_file_mocks.MockPitFileConverter{} + converter.SetConverterError(fakes.FakeError) + fetcher := &mocks.MockLogFetcher{} + fetcher.SetFetchedLogs([]types.Log{test_data.EthPitFileLog}) + repository := &pit_file_mocks.MockPitFileRepository{} + repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) + transformer := pit_file.PitFileTransformer{ + Fetcher: fetcher, + Converter: converter, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + + It("persists pit file model", func() { + converter := &pit_file_mocks.MockPitFileConverter{} + fetcher := &mocks.MockLogFetcher{} + fetcher.SetFetchedLogs([]types.Log{test_data.EthPitFileLog}) + repository := &pit_file_mocks.MockPitFileRepository{} + fakeHeader := core.Header{BlockNumber: 1, Id: 2} + repository.SetMissingHeaders([]core.Header{fakeHeader}) + transformer := pit_file.PitFileTransformer{ + Fetcher: fetcher, + Converter: converter, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id)) + Expect(repository.PassedTransactionIndex).To(Equal(test_data.EthPitFileLog.TxIndex)) + Expect(repository.PassedModel).To(Equal(test_data.PitFileModel)) + }) + + It("returns error if repository returns error for create", func() { + converter := &pit_file_mocks.MockPitFileConverter{} + fetcher := &mocks.MockLogFetcher{} + fetcher.SetFetchedLogs([]types.Log{test_data.EthPitFileLog}) + repository := &pit_file_mocks.MockPitFileRepository{} + repository.SetMissingHeaders([]core.Header{{BlockNumber: 1, Id: 2}}) + repository.SetCreateError(fakes.FakeError) + transformer := pit_file.PitFileTransformer{ + Fetcher: fetcher, + Converter: converter, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) +}) diff --git a/pkg/transformers/shared/constants.go b/pkg/transformers/shared/constants.go index 938a79f4..e1fda760 100644 --- a/pkg/transformers/shared/constants.go +++ b/pkg/transformers/shared/constants.go @@ -15,9 +15,11 @@ package shared var ( + FlipperABI = `[{"constant":true,"inputs":[{"name":"","type":"uint256"}],"name":"bids","outputs":[{"name":"bid","type":"uint256"},{"name":"lot","type":"uint256"},{"name":"guy","type":"address"},{"name":"tic","type":"uint48"},{"name":"end","type":"uint48"},{"name":"urn","type":"bytes32"},{"name":"gal","type":"address"},{"name":"tab","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function","signature":"0x4423c5f1"},{"constant":true,"inputs":[],"name":"ttl","outputs":[{"name":"","type":"uint48"}],"payable":false,"stateMutability":"view","type":"function","signature":"0x4e8b1dd5"},{"constant":true,"inputs":[],"name":"gem","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function","signature":"0x7bd2bea7"},{"constant":true,"inputs":[],"name":"beg","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function","signature":"0x7d780d82"},{"constant":true,"inputs":[],"name":"tau","outputs":[{"name":"","type":"uint48"}],"payable":false,"stateMutability":"view","type":"function","signature":"0xcfc4af55"},{"constant":true,"inputs":[],"name":"kicks","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function","signature":"0xcfdd3302"},{"constant":true,"inputs":[],"name":"dai","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function","signature":"0xf4b9fa75"},{"inputs":[{"name":"dai_","type":"address"},{"name":"gem_","type":"address"}],"payable":false,"stateMutability":"nonpayable","type":"constructor","signature":"constructor"},{"anonymous":false,"inputs":[{"indexed":true,"name":"id","type":"uint256"},{"indexed":false,"name":"lot","type":"uint256"},{"indexed":false,"name":"bid","type":"uint256"},{"indexed":false,"name":"gal","type":"address"},{"indexed":false,"name":"end","type":"uint48"},{"indexed":true,"name":"urn","type":"bytes32"},{"indexed":false,"name":"tab","type":"uint256"}],"name":"Kick","type":"event","signature":"0xbac86238bdba81d21995024470425ecb370078fa62b7271b90cf28cbd1e3e87e"},{"anonymous":true,"inputs":[{"indexed":true,"name":"sig","type":"bytes4"},{"indexed":true,"name":"guy","type":"address"},{"indexed":true,"name":"foo","type":"bytes32"},{"indexed":true,"name":"bar","type":"bytes32"},{"indexed":false,"name":"wad","type":"uint256"},{"indexed":false,"name":"fax","type":"bytes"}],"name":"LogNote","type":"event","signature":"0x644843f351d3fba4abcd60109eaff9f54bac8fb8ccf0bab941009c21df21cf31"},{"constant":true,"inputs":[],"name":"era","outputs":[{"name":"","type":"uint48"}],"payable":false,"stateMutability":"view","type":"function","signature":"0x143e55e0"},{"constant":false,"inputs":[{"name":"urn","type":"bytes32"},{"name":"gal","type":"address"},{"name":"tab","type":"uint256"},{"name":"lot","type":"uint256"},{"name":"bid","type":"uint256"}],"name":"kick","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"nonpayable","type":"function","signature":"0xeae19d9e"},{"constant":false,"inputs":[{"name":"id","type":"uint256"}],"name":"tick","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function","signature":"0xfc7b6aee"},{"constant":false,"inputs":[{"name":"id","type":"uint256"},{"name":"lot","type":"uint256"},{"name":"bid","type":"uint256"}],"name":"tend","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function","signature":"0x4b43ed12"},{"constant":false,"inputs":[{"name":"id","type":"uint256"},{"name":"lot","type":"uint256"},{"name":"bid","type":"uint256"}],"name":"dent","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function","signature":"0x5ff3a382"},{"constant":false,"inputs":[{"name":"id","type":"uint256"}],"name":"deal","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function","signature":"0xc959c42b"}]` FlipperContractAddress = "0x6b59c42097e2fff7cad96cb08ceefd601081ad9c" //this is a temporary address deployed locally - FlipperABI = "[{\"constant\":true,\"inputs\":[],\"name\":\"era\",\"outputs\":[{\"name\":\"\",\"type\":\"uint48\"}],\"payable\":false,\"stateMutability\":\"view\",\"type\":\"function\"},{\"constant\":true,\"inputs\":[{\"name\":\"\",\"type\":\"uint256\"}],\"name\":\"bids\",\"outputs\":[{\"name\":\"bid\",\"type\":\"uint256\"},{\"name\":\"lot\",\"type\":\"uint256\"},{\"name\":\"guy\",\"type\":\"address\"},{\"name\":\"tic\",\"type\":\"uint48\"},{\"name\":\"end\",\"type\":\"uint48\"},{\"name\":\"urn\",\"type\":\"bytes32\"},{\"name\":\"gal\",\"type\":\"address\"},{\"name\":\"tab\",\"type\":\"uint256\"}],\"payable\":false,\"stateMutability\":\"view\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"name\":\"id\",\"type\":\"uint256\"},{\"name\":\"lot\",\"type\":\"uint256\"},{\"name\":\"bid\",\"type\":\"uint256\"}],\"name\":\"tend\",\"outputs\":[],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":true,\"inputs\":[],\"name\":\"ttl\",\"outputs\":[{\"name\":\"\",\"type\":\"uint48\"}],\"payable\":false,\"stateMutability\":\"view\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"name\":\"id\",\"type\":\"uint256\"},{\"name\":\"lot\",\"type\":\"uint256\"},{\"name\":\"bid\",\"type\":\"uint256\"}],\"name\":\"dent\",\"outputs\":[],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":true,\"inputs\":[],\"name\":\"gem\",\"outputs\":[{\"name\":\"\",\"type\":\"address\"}],\"payable\":false,\"stateMutability\":\"view\",\"type\":\"function\"},{\"constant\":true,\"inputs\":[],\"name\":\"beg\",\"outputs\":[{\"name\":\"\",\"type\":\"uint256\"}],\"payable\":false,\"stateMutability\":\"view\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"name\":\"id\",\"type\":\"uint256\"}],\"name\":\"deal\",\"outputs\":[],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":true,\"inputs\":[],\"name\":\"tau\",\"outputs\":[{\"name\":\"\",\"type\":\"uint48\"}],\"payable\":false,\"stateMutability\":\"view\",\"type\":\"function\"},{\"constant\":true,\"inputs\":[],\"name\":\"kicks\",\"outputs\":[{\"name\":\"\",\"type\":\"uint256\"}],\"payable\":false,\"stateMutability\":\"view\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"name\":\"urn\",\"type\":\"bytes32\"},{\"name\":\"gal\",\"type\":\"address\"},{\"name\":\"tab\",\"type\":\"uint256\"},{\"name\":\"lot\",\"type\":\"uint256\"},{\"name\":\"bid\",\"type\":\"uint256\"}],\"name\":\"kick\",\"outputs\":[{\"name\":\"\",\"type\":\"uint256\"}],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":true,\"inputs\":[],\"name\":\"dai\",\"outputs\":[{\"name\":\"\",\"type\":\"address\"}],\"payable\":false,\"stateMutability\":\"view\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"name\":\"id\",\"type\":\"uint256\"}],\"name\":\"tick\",\"outputs\":[],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"name\":\"dai_\",\"type\":\"address\"},{\"name\":\"gem_\",\"type\":\"address\"}],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"constructor\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"name\":\"id\",\"type\":\"uint256\"},{\"indexed\":false,\"name\":\"lot\",\"type\":\"uint256\"},{\"indexed\":false,\"name\":\"bid\",\"type\":\"uint256\"},{\"indexed\":false,\"name\":\"gal\",\"type\":\"address\"},{\"indexed\":false,\"name\":\"end\",\"type\":\"uint48\"},{\"indexed\":true,\"name\":\"urn\",\"type\":\"bytes32\"},{\"indexed\":false,\"name\":\"tab\",\"type\":\"uint256\"}],\"name\":\"Kick\",\"type\":\"event\"},{\"anonymous\":true,\"inputs\":[{\"indexed\":true,\"name\":\"sig\",\"type\":\"bytes4\"},{\"indexed\":true,\"name\":\"guy\",\"type\":\"address\"},{\"indexed\":true,\"name\":\"foo\",\"type\":\"bytes32\"},{\"indexed\":true,\"name\":\"bar\",\"type\":\"bytes32\"},{\"indexed\":false,\"name\":\"wad\",\"type\":\"uint256\"},{\"indexed\":false,\"name\":\"fax\",\"type\":\"bytes\"}],\"name\":\"LogNote\",\"type\":\"event\"}]" - FlipKickSignature = "0xbac86238bdba81d21995024470425ecb370078fa62b7271b90cf28cbd1e3e87e" - BiteSignature = "0x44a8f29dfcf27e5dd7a4db50b390f851bf3ecf2bf221243be3e853f586134455" + FlipKickSignature = "0x8828a22eb6a18623309ad55592866c4b077989e9e8a25e1b85f9bf6f7282520f" + PitABI = `[{"constant":true,"inputs":[],"name":"vat","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"live","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"drip","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"Line","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"","type":"address"}],"name":"wards","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"","type":"bytes32"}],"name":"ilks","outputs":[{"name":"spot","type":"uint256"},{"name":"line","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"inputs":[{"name":"vat_","type":"address"}],"payable":false,"stateMutability":"nonpayable","type":"constructor"},{"anonymous":false,"inputs":[{"indexed":true,"name":"ilk","type":"bytes32"},{"indexed":true,"name":"lad","type":"bytes32"},{"indexed":false,"name":"ink","type":"uint256"},{"indexed":false,"name":"art","type":"uint256"},{"indexed":false,"name":"dink","type":"int256"},{"indexed":false,"name":"dart","type":"int256"},{"indexed":false,"name":"iArt","type":"uint256"}],"name":"Frob","type":"event"},{"anonymous":true,"inputs":[{"indexed":true,"name":"sig","type":"bytes4"},{"indexed":true,"name":"guy","type":"address"},{"indexed":true,"name":"foo","type":"bytes32"},{"indexed":true,"name":"bar","type":"bytes32"},{"indexed":false,"name":"wad","type":"uint256"},{"indexed":false,"name":"fax","type":"bytes"}],"name":"LogNote","type":"event"},{"constant":false,"inputs":[{"name":"guy","type":"address"}],"name":"rely","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"guy","type":"address"}],"name":"deny","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"ilk","type":"bytes32"},{"name":"what","type":"bytes32"},{"name":"data","type":"uint256"}],"name":"file","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"what","type":"bytes32"},{"name":"data","type":"uint256"}],"name":"file","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"what","type":"bytes32"},{"name":"data","type":"address"}],"name":"file","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"ilk","type":"bytes32"},{"name":"dink","type":"int256"},{"name":"dart","type":"int256"}],"name":"frob","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"}]` + PitFileSignature = "0x1a0b287e00000000000000000000000000000000000000000000000000000000" TendFunctionSignature = "0x4b43ed1200000000000000000000000000000000000000000000000000000000" + BiteSignature = "0x44a8f29dfcf27e5dd7a4db50b390f851bf3ecf2bf221243be3e853f586134455" ) diff --git a/pkg/transformers/shared/transformer.go b/pkg/transformers/shared/transformer.go index 56f8fc40..9632e6cd 100644 --- a/pkg/transformers/shared/transformer.go +++ b/pkg/transformers/shared/transformer.go @@ -28,7 +28,7 @@ type Transformer interface { type TransformerInitializer func(db *postgres.DB, blockChain core.BlockChain) Transformer type TransformerConfig struct { - ContractAddresses string + ContractAddress string ContractAbi string Topics []string StartingBlockNumber int64 diff --git a/pkg/transformers/tend/config.go b/pkg/transformers/tend/config.go index 3859186d..f8fcc847 100644 --- a/pkg/transformers/tend/config.go +++ b/pkg/transformers/tend/config.go @@ -17,7 +17,7 @@ package tend import "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" var TendConfig = shared.TransformerConfig{ - ContractAddresses: shared.FlipperContractAddress, + ContractAddress: shared.FlipperContractAddress, ContractAbi: shared.FlipperABI, Topics: []string{shared.TendFunctionSignature}, StartingBlockNumber: 0, diff --git a/pkg/transformers/tend/transformer.go b/pkg/transformers/tend/transformer.go index e1b9864e..89b285c8 100644 --- a/pkg/transformers/tend/transformer.go +++ b/pkg/transformers/tend/transformer.go @@ -59,14 +59,14 @@ func (t TendTransformer) Execute() error { } for _, header := range missingHeaders { - ethLogs, err := t.Fetcher.FetchLogs(config.ContractAddresses, topics, header.BlockNumber) + ethLogs, err := t.Fetcher.FetchLogs(config.ContractAddress, topics, header.BlockNumber) if err != nil { log.Println("Error fetching matching logs:", err) return err } for _, ethLog := range ethLogs { - model, err := t.Converter.Convert(config.ContractAddresses, config.ContractAbi, ethLog) + model, err := t.Converter.Convert(config.ContractAddress, config.ContractAbi, ethLog) if err != nil { log.Println("Error converting logs:", err) return err diff --git a/pkg/transformers/tend/transformer_test.go b/pkg/transformers/tend/transformer_test.go index 3fa3ae58..d3fd3570 100644 --- a/pkg/transformers/tend/transformer_test.go +++ b/pkg/transformers/tend/transformer_test.go @@ -93,7 +93,7 @@ var _ = Describe("Tend Transformer", func() { err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(converter.ConverterContract).To(Equal(tend.TendConfig.ContractAddresses)) + Expect(converter.ConverterContract).To(Equal(tend.TendConfig.ContractAddress)) Expect(converter.ConverterAbi).To(Equal(tend.TendConfig.ContractAbi)) Expect(converter.LogsToConvert).To(Equal([]types.Log{test_data.TendLogNote})) }) diff --git a/pkg/transformers/test_data/mocks/pit_file/converter.go b/pkg/transformers/test_data/mocks/pit_file/converter.go new file mode 100644 index 00000000..45693ea4 --- /dev/null +++ b/pkg/transformers/test_data/mocks/pit_file/converter.go @@ -0,0 +1,26 @@ +package pit_file + +import ( + "github.com/ethereum/go-ethereum/core/types" + + "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file" + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" +) + +type MockPitFileConverter struct { + PassedContractAddress string + PassedContractABI string + PassedLog types.Log + converterError error +} + +func (converter *MockPitFileConverter) SetConverterError(err error) { + converter.converterError = err +} + +func (converter *MockPitFileConverter) ToModel(contractAddress string, contractAbi string, ethLog types.Log) (pit_file.PitFileModel, error) { + converter.PassedContractAddress = contractAddress + converter.PassedContractABI = contractAbi + converter.PassedLog = ethLog + return test_data.PitFileModel, converter.converterError +} diff --git a/pkg/transformers/test_data/mocks/pit_file/repository.go b/pkg/transformers/test_data/mocks/pit_file/repository.go new file mode 100644 index 00000000..9475adaa --- /dev/null +++ b/pkg/transformers/test_data/mocks/pit_file/repository.go @@ -0,0 +1,42 @@ +package pit_file + +import ( + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file" +) + +type MockPitFileRepository struct { + createError error + PassedEndingBlockNumber int64 + PassedModel pit_file.PitFileModel + PassedHeaderID int64 + PassedStartingBlockNumber int64 + PassedTransactionIndex uint + missingHeaders []core.Header + missingHeadersErr error +} + +func (repository *MockPitFileRepository) SetCreateError(err error) { + repository.createError = err +} + +func (repository *MockPitFileRepository) SetMissingHeadersErr(err error) { + repository.missingHeadersErr = err +} + +func (repository *MockPitFileRepository) SetMissingHeaders(headers []core.Header) { + repository.missingHeaders = headers +} + +func (repository *MockPitFileRepository) Create(headerID int64, transactionIndex uint, model pit_file.PitFileModel) error { + repository.PassedHeaderID = headerID + repository.PassedTransactionIndex = transactionIndex + repository.PassedModel = model + return repository.createError +} + +func (repository *MockPitFileRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + repository.PassedStartingBlockNumber = startingBlockNumber + repository.PassedEndingBlockNumber = endingBlockNumber + return repository.missingHeaders, repository.missingHeadersErr +} diff --git a/pkg/transformers/test_data/pit_file.go b/pkg/transformers/test_data/pit_file.go new file mode 100644 index 00000000..db1a0c0a --- /dev/null +++ b/pkg/transformers/test_data/pit_file.go @@ -0,0 +1,41 @@ +package test_data + +import ( + "encoding/json" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file" + "math/big" +) + +var ( + PitAddress = "0xff3f2400f1600f3f493a9a92704a29b96795af1a" +) + +var EthPitFileLog = types.Log{ + Address: common.HexToAddress(PitAddress), + Topics: []common.Hash{ + common.HexToHash("0x1a0b287e00000000000000000000000000000000000000000000000000000000"), + common.HexToHash("0x0000000000000000000000000f243e26db94b5426032e6dfa6007802dea2a614"), + common.HexToHash("0x66616b6520696c6b000000000000000000000000000000000000000000000000"), + common.HexToHash("0x73706f7400000000000000000000000000000000000000000000000000000000"), + }, + Data: hexutil.MustDecode("0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000641a0b287e66616b6520696c6b00000000000000000000000000000000000000000000000073706f7400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007b"), + BlockNumber: 11, + TxHash: common.HexToHash("0x1ba8125f60fa045c85b35df3983bee37db8627fbc32e3442a5cf17c85bb83f09"), + TxIndex: 111, + BlockHash: common.HexToHash("0x6dc284247c524b22b10a75ef1c9d1709a509208d04c15fa2b675a293db637d21"), + Index: 0, + Removed: false, +} + +var raw, _ = json.Marshal(EthPitFileLog) + +var PitFileModel = pit_file.PitFileModel{ + Ilk: "fake ilk", + What: "spot", + Risk: big.NewInt(123).String(), + TransactionIndex: EthPitFileLog.TxIndex, + Raw: raw, +} diff --git a/pkg/transformers/transformers.go b/pkg/transformers/transformers.go index 7278e643..a8af1a44 100644 --- a/pkg/transformers/transformers.go +++ b/pkg/transformers/transformers.go @@ -18,6 +18,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/bite" "github.com/vulcanize/vulcanizedb/pkg/transformers/flip_kick" "github.com/vulcanize/vulcanizedb/pkg/transformers/frob" + "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file" "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/tend" @@ -28,6 +29,8 @@ func TransformerInitializers() []shared.TransformerInitializer { flipKickTransformerInitializer := flip_kick.FlipKickTransformerInitializer{Config: flipKickConfig} frobConfig := frob.FrobConfig frobTransformerInitializer := frob.FrobTransformerInitializer{Config: frobConfig} + pitFileConfig := pit_file.PitFileConfig + pitFileTransformerInitializer := pit_file.PitFileTransformerInitializer{Config: pitFileConfig} priceFeedConfig := price_feeds.PriceFeedConfig priceFeedTransformerInitializer := price_feeds.PriceFeedTransformerInitializer{Config: priceFeedConfig} tendConfig := tend.TendConfig @@ -38,6 +41,7 @@ func TransformerInitializers() []shared.TransformerInitializer { biteTransformerInitializer.NewBiteTransformer, flipKickTransformerInitializer.NewFlipKickTransformer, frobTransformerInitializer.NewFrobTransformer, + pitFileTransformerInitializer.NewPitFileTransformer, priceFeedTransformerInitializer.NewPriceFeedTransformer, tendTransformerInitializer.NewTendTransformer, } diff --git a/test_config/test_config.go b/test_config/test_config.go index e6926a82..29bf6ca1 100644 --- a/test_config/test_config.go +++ b/test_config/test_config.go @@ -77,6 +77,7 @@ func CleanTestDB(db *postgres.DB) { db.MustExec("DELETE FROM headers") db.MustExec("DELETE FROM log_filters") db.MustExec("DELETE FROM logs") + db.MustExec("DELETE FROM maker.pit_file") db.MustExec("DELETE FROM receipts") db.MustExec("DELETE FROM transactions") db.MustExec("DELETE FROM watched_contracts")