From 939ead0c8234e3342bbdd5c275cf7db8a4df859b Mon Sep 17 00:00:00 2001 From: Elizabeth Date: Thu, 13 Sep 2018 09:06:26 -0500 Subject: [PATCH] Deal transformer (#19) --- .../1536615440_create_deal_table.down.sql | 1 + .../1536615440_create_deal_table.up.sql | 8 + db/schema.sql | 64 ++++++++ pkg/transformers/deal/config.go | 25 ++++ pkg/transformers/deal/converter.go | 58 ++++++++ pkg/transformers/deal/converter_test.go | 45 ++++++ pkg/transformers/deal/deal_suite_test.go | 19 +++ pkg/transformers/deal/model.go | 21 +++ pkg/transformers/deal/repository.go | 56 +++++++ pkg/transformers/deal/repository_test.go | 138 ++++++++++++++++++ pkg/transformers/deal/transformer.go | 77 ++++++++++ pkg/transformers/deal/transformer_test.go | 120 +++++++++++++++ pkg/transformers/dent/converter.go | 4 +- pkg/transformers/dent/converter_test.go | 7 +- pkg/transformers/dent/transformer.go | 2 +- pkg/transformers/dent/transformer_test.go | 2 - pkg/transformers/shared/constants.go | 2 + .../shared/event_signature_generator_test.go | 6 + pkg/transformers/test_data/deal.go | 48 ++++++ .../test_data/mocks/deal/converter.go | 38 +++++ .../test_data/mocks/deal/repository.go | 54 +++++++ .../test_data/mocks/dent/converter.go | 4 +- pkg/transformers/transformers.go | 6 +- 23 files changed, 791 insertions(+), 14 deletions(-) create mode 100644 db/migrations/1536615440_create_deal_table.down.sql create mode 100644 db/migrations/1536615440_create_deal_table.up.sql create mode 100644 pkg/transformers/deal/config.go create mode 100644 pkg/transformers/deal/converter.go create mode 100644 pkg/transformers/deal/converter_test.go create mode 100644 pkg/transformers/deal/deal_suite_test.go create mode 100644 pkg/transformers/deal/model.go create mode 100644 pkg/transformers/deal/repository.go create mode 100644 pkg/transformers/deal/repository_test.go create mode 100644 pkg/transformers/deal/transformer.go create mode 100644 pkg/transformers/deal/transformer_test.go create mode 100644 pkg/transformers/test_data/deal.go create mode 100644 pkg/transformers/test_data/mocks/deal/converter.go create mode 100644 pkg/transformers/test_data/mocks/deal/repository.go diff --git a/db/migrations/1536615440_create_deal_table.down.sql b/db/migrations/1536615440_create_deal_table.down.sql new file mode 100644 index 00000000..0080a2d6 --- /dev/null +++ b/db/migrations/1536615440_create_deal_table.down.sql @@ -0,0 +1 @@ +DROP TABLE maker.deal; diff --git a/db/migrations/1536615440_create_deal_table.up.sql b/db/migrations/1536615440_create_deal_table.up.sql new file mode 100644 index 00000000..2950bb53 --- /dev/null +++ b/db/migrations/1536615440_create_deal_table.up.sql @@ -0,0 +1,8 @@ +CREATE TABLE maker.deal ( + id SERIAL PRIMARY KEY, + header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE, + bid_id NUMERIC NOT NULL, + tx_idx INTEGER NOT NUll, + raw_log JSONB, + UNIQUE (header_id, tx_idx) +); \ No newline at end of file diff --git a/db/schema.sql b/db/schema.sql index 09664db5..b19de60b 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -79,6 +79,39 @@ CREATE SEQUENCE maker.bite_id_seq ALTER SEQUENCE maker.bite_id_seq OWNED BY maker.bite.id; +-- +-- Name: deal; Type: TABLE; Schema: maker; Owner: - +-- + +CREATE TABLE maker.deal ( + id integer NOT NULL, + header_id integer NOT NULL, + bid_id numeric NOT NULL, + tx_idx integer NOT NULL, + raw_log jsonb +); + + +-- +-- Name: deal_id_seq; Type: SEQUENCE; Schema: maker; Owner: - +-- + +CREATE SEQUENCE maker.deal_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: deal_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: - +-- + +ALTER SEQUENCE maker.deal_id_seq OWNED BY maker.deal.id; + + -- -- Name: dent; Type: TABLE; Schema: maker; Owner: - -- @@ -884,6 +917,13 @@ CREATE VIEW public.watched_event_logs AS ALTER TABLE ONLY maker.bite ALTER COLUMN id SET DEFAULT nextval('maker.bite_id_seq'::regclass); +-- +-- Name: deal id; Type: DEFAULT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.deal ALTER COLUMN id SET DEFAULT nextval('maker.deal_id_seq'::regclass); + + -- -- Name: dent db_id; Type: DEFAULT; Schema: maker; Owner: - -- @@ -1047,6 +1087,22 @@ ALTER TABLE ONLY maker.bite ADD CONSTRAINT bite_pkey PRIMARY KEY (id); +-- +-- Name: deal deal_header_id_tx_idx_key; Type: CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.deal + ADD CONSTRAINT deal_header_id_tx_idx_key UNIQUE (header_id, tx_idx); + + +-- +-- Name: deal deal_pkey; Type: CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.deal + ADD CONSTRAINT deal_pkey PRIMARY KEY (id); + + -- -- Name: dent dent_bid_id_key; Type: CONSTRAINT; Schema: maker; Owner: - -- @@ -1378,6 +1434,14 @@ ALTER TABLE ONLY maker.bite ADD CONSTRAINT bite_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE; +-- +-- Name: deal deal_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.deal + ADD CONSTRAINT deal_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE; + + -- -- Name: dent dent_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - -- diff --git a/pkg/transformers/deal/config.go b/pkg/transformers/deal/config.go new file mode 100644 index 00000000..621921b7 --- /dev/null +++ b/pkg/transformers/deal/config.go @@ -0,0 +1,25 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package deal + +import "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + +var Config = shared.TransformerConfig{ + ContractAddress: shared.FlipperContractAddress, + ContractAbi: shared.FlipperABI, + Topics: []string{shared.DealSignature}, + StartingBlockNumber: 0, + EndingBlockNumber: 100, +} diff --git a/pkg/transformers/deal/converter.go b/pkg/transformers/deal/converter.go new file mode 100644 index 00000000..fffda063 --- /dev/null +++ b/pkg/transformers/deal/converter.go @@ -0,0 +1,58 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package deal + +import ( + "encoding/json" + "errors" + + "github.com/ethereum/go-ethereum/core/types" +) + +type Converter interface { + ToModel(ethLog types.Log) (DealModel, error) +} + +type DealConverter struct{} + +func NewDealConverter() DealConverter { + return DealConverter{} +} + +func (DealConverter) ToModel(ethLog types.Log) (DealModel, error) { + err := validateLog(ethLog) + if err != nil { + return DealModel{}, err + } + + bidId := ethLog.Topics[2].Big() + raw, err := json.Marshal(ethLog) + if err != nil { + return DealModel{}, err + } + + return DealModel{ + BidId: bidId.String(), + TransactionIndex: ethLog.TxIndex, + Raw: raw, + }, nil +} + +func validateLog(ethLog types.Log) error { + if len(ethLog.Topics) < 3 { + return errors.New("deal log does not contain expected topics") + } + return nil +} diff --git a/pkg/transformers/deal/converter_test.go b/pkg/transformers/deal/converter_test.go new file mode 100644 index 00000000..f28478cd --- /dev/null +++ b/pkg/transformers/deal/converter_test.go @@ -0,0 +1,45 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package deal_test + +import ( + "github.com/ethereum/go-ethereum/common" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/transformers/deal" + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" +) + +var _ = Describe("Flip Deal Converter", func() { + It("converts a log to a model", func() { + converter := deal.DealConverter{} + + model, err := converter.ToModel(test_data.DealLogNote) + + Expect(err).NotTo(HaveOccurred()) + Expect(model).To(Equal(test_data.DealModel)) + }) + + It("returns an error if the expected amount of topics aren't in the log", func() { + converter := deal.DealConverter{} + invalidLog := test_data.DealLogNote + invalidLog.Topics = []common.Hash{} + model, err := converter.ToModel(invalidLog) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError("deal log does not contain expected topics")) + Expect(model).To(Equal(deal.DealModel{})) + }) +}) diff --git a/pkg/transformers/deal/deal_suite_test.go b/pkg/transformers/deal/deal_suite_test.go new file mode 100644 index 00000000..1b987194 --- /dev/null +++ b/pkg/transformers/deal/deal_suite_test.go @@ -0,0 +1,19 @@ +package deal_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "io/ioutil" + "log" +) + +func TestFlipDeal(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Deal Suite") +} + +var _ = BeforeSuite(func() { + log.SetOutput(ioutil.Discard) +}) diff --git a/pkg/transformers/deal/model.go b/pkg/transformers/deal/model.go new file mode 100644 index 00000000..c434b45a --- /dev/null +++ b/pkg/transformers/deal/model.go @@ -0,0 +1,21 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package deal + +type DealModel struct { + BidId string `db:"bid_id"` + TransactionIndex uint `db:"tx_idx"` + Raw []byte `db:"raw_log"` +} diff --git a/pkg/transformers/deal/repository.go b/pkg/transformers/deal/repository.go new file mode 100644 index 00000000..4f53467b --- /dev/null +++ b/pkg/transformers/deal/repository.go @@ -0,0 +1,56 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package deal + +import ( + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" +) + +type Repository interface { + Create(headerId int64, model DealModel) error + MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) +} +type DealRepository struct { + db *postgres.DB +} + +func NewDealRepository(database *postgres.DB) DealRepository { + return DealRepository{db: database} +} +func (r DealRepository) Create(headerId int64, model DealModel) error { + _, err := r.db.Exec( + `INSERT into maker.deal (header_id, bid_id, tx_idx, raw_log) + VALUES($1, $2, $3, $4)`, + headerId, model.BidId, model.TransactionIndex, model.Raw, + ) + return err +} +func (r DealRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + var missingHeaders []core.Header + err := r.db.Select( + &missingHeaders, + `SELECT headers.id, headers.block_number FROM headers + LEFT JOIN maker.deal on headers.id = header_id + WHERE header_id ISNULL + AND headers.block_number >= $1 + AND headers.block_number <= $2 + AND headers.eth_node_fingerprint = $3`, + startingBlockNumber, + endingBlockNumber, + r.db.Node.ID, + ) + return missingHeaders, err +} diff --git a/pkg/transformers/deal/repository_test.go b/pkg/transformers/deal/repository_test.go new file mode 100644 index 00000000..4bb8539d --- /dev/null +++ b/pkg/transformers/deal/repository_test.go @@ -0,0 +1,138 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package deal_test + +import ( + "math/rand" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/vulcanize/vulcanizedb/pkg/transformers/deal" + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" + "github.com/vulcanize/vulcanizedb/test_config" +) + +var _ = Describe("Deal Repository", func() { + var node core.Node + var db *postgres.DB + var dealRepository deal.DealRepository + var headerRepository repositories.HeaderRepository + var headerId int64 + var err error + + BeforeEach(func() { + node = test_config.NewTestNode() + db = test_config.NewTestDB(node) + test_config.CleanTestDB(db) + dealRepository = deal.NewDealRepository(db) + headerRepository = repositories.NewHeaderRepository(db) + }) + Describe("Create", func() { + BeforeEach(func() { + headerId, err = headerRepository.CreateOrUpdateHeader(core.Header{}) + Expect(err).NotTo(HaveOccurred()) + err := dealRepository.Create(headerId, test_data.DealModel) + Expect(err).NotTo(HaveOccurred()) + }) + + It("persists a deal record", func() { + var count int + db.QueryRow(`SELECT count(*) FROM maker.deal`).Scan(&count) + Expect(count).To(Equal(1)) + var dbResult deal.DealModel + err = db.Get(&dbResult, `SELECT bid_id, tx_idx, raw_log FROM maker.deal WHERE header_id = $1`, headerId) + Expect(err).NotTo(HaveOccurred()) + Expect(dbResult.BidId).To(Equal(test_data.DealModel.BidId)) + Expect(dbResult.TransactionIndex).To(Equal(test_data.DealModel.TransactionIndex)) + Expect(dbResult.Raw).To(MatchJSON(test_data.DealModel.Raw)) + }) + + It("returns an error if inserting a deal record fails", func() { + err = dealRepository.Create(headerId, test_data.DealModel) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint")) + }) + + It("deletes the deal record if its corresponding header record is deleted", func() { + var count int + err = db.QueryRow(`SELECT count(*) from maker.deal`).Scan(&count) + Expect(err).NotTo(HaveOccurred()) + Expect(count).To(Equal(1)) + _, err = db.Exec(`DELETE FROM headers where id = $1`, headerId) + Expect(err).NotTo(HaveOccurred()) + err = db.QueryRow(`SELECT count(*) from maker.deal`).Scan(&count) + Expect(err).NotTo(HaveOccurred()) + Expect(count).To(Equal(0)) + }) + }) + + Describe("MissingHeaders", func() { + var dealBlockNumber int64 + var startingBlockNumber int64 + var endingBlockNumber int64 + var blockNumbers []int64 + + BeforeEach(func() { + dealBlockNumber = rand.Int63() + startingBlockNumber = dealBlockNumber - 1 + endingBlockNumber = dealBlockNumber + 1 + outOfRangeBlockNumber := dealBlockNumber + 2 + blockNumbers = []int64{startingBlockNumber, dealBlockNumber, endingBlockNumber, outOfRangeBlockNumber} + var headerIds []int64 + for _, number := range blockNumbers { + headerId, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: number}) + Expect(err).NotTo(HaveOccurred()) + headerIds = append(headerIds, headerId) + } + dealRepository.Create(headerIds[1], test_data.DealModel) + }) + + It("returns header records that don't have a corresponding deals", func() { + missingHeaders, err := dealRepository.MissingHeaders(startingBlockNumber, endingBlockNumber) + Expect(err).NotTo(HaveOccurred()) + Expect(len(missingHeaders)).To(Equal(2)) + Expect(missingHeaders[0].BlockNumber).To(Equal(startingBlockNumber)) + Expect(missingHeaders[1].BlockNumber).To(Equal(endingBlockNumber)) + }) + + It("only returns missing headers for the given node", func() { + node2 := core.Node{} + db2 := test_config.NewTestDB(node2) + dealRepository2 := deal.NewDealRepository(db2) + headerRepository2 := repositories.NewHeaderRepository(db2) + var node2HeaderIds []int64 + for _, number := range blockNumbers { + id, err := headerRepository2.CreateOrUpdateHeader(core.Header{BlockNumber: number}) + node2HeaderIds = append(node2HeaderIds, id) + Expect(err).NotTo(HaveOccurred()) + } + missingHeadersNode1, err := dealRepository.MissingHeaders(startingBlockNumber, endingBlockNumber) + Expect(err).NotTo(HaveOccurred()) + Expect(len(missingHeadersNode1)).To(Equal(2)) + Expect(missingHeadersNode1[0].BlockNumber).To(Equal(startingBlockNumber)) + Expect(missingHeadersNode1[1].BlockNumber).To(Equal(endingBlockNumber)) + missingHeadersNode2, err := dealRepository2.MissingHeaders(startingBlockNumber, endingBlockNumber) + Expect(err).NotTo(HaveOccurred()) + Expect(len(missingHeadersNode2)).To(Equal(3)) + Expect(missingHeadersNode2[0].BlockNumber).To(Equal(startingBlockNumber)) + Expect(missingHeadersNode2[1].BlockNumber).To(Equal(dealBlockNumber)) + Expect(missingHeadersNode2[2].BlockNumber).To(Equal(endingBlockNumber)) + }) + }) +}) diff --git a/pkg/transformers/deal/transformer.go b/pkg/transformers/deal/transformer.go new file mode 100644 index 00000000..435d2e13 --- /dev/null +++ b/pkg/transformers/deal/transformer.go @@ -0,0 +1,77 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package deal + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "log" +) + +type DealTransformer struct { + Config shared.TransformerConfig + Converter Converter + Fetcher shared.LogFetcher + Repository Repository +} + +type DealTransformerInitializer struct { + Config shared.TransformerConfig +} + +func (i DealTransformerInitializer) NewDealTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer { + converter := NewDealConverter() + fetcher := shared.NewFetcher(blockChain) + repository := NewDealRepository(db) + return DealTransformer{ + Config: i.Config, + Converter: converter, + Fetcher: fetcher, + Repository: repository, + } +} + +func (t DealTransformer) Execute() error { + config := t.Config + topics := [][]common.Hash{{common.HexToHash(shared.DealSignature)}} + + headers, err := t.Repository.MissingHeaders(config.StartingBlockNumber, config.EndingBlockNumber) + if err != nil { + return err + } + + for _, header := range headers { + ethLogs, err := t.Fetcher.FetchLogs(config.ContractAddress, topics, header.BlockNumber) + if err != nil { + log.Println("Error fetching deal logs:", err) + return err + } + for _, ethLog := range ethLogs { + model, err := t.Converter.ToModel(ethLog) + if err != nil { + log.Println("Error converting deal log", err) + return err + } + err = t.Repository.Create(header.Id, model) + if err != nil { + log.Println("Error persisting deal record", err) + return err + } + } + } + return err +} diff --git a/pkg/transformers/deal/transformer_test.go b/pkg/transformers/deal/transformer_test.go new file mode 100644 index 00000000..95c0e7b5 --- /dev/null +++ b/pkg/transformers/deal/transformer_test.go @@ -0,0 +1,120 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package deal_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/fakes" + "github.com/vulcanize/vulcanizedb/pkg/transformers/deal" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks" + deal_mocks "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks/deal" + "math/rand" +) + +var _ = Describe("DealTransformer", func() { + var config = deal.Config + var dealRepository deal_mocks.MockDealRepository + var fetcher mocks.MockLogFetcher + var converter deal_mocks.MockDealConverter + var transformer deal.DealTransformer + + BeforeEach(func() { + dealRepository = deal_mocks.MockDealRepository{} + fetcher = mocks.MockLogFetcher{} + converter = deal_mocks.MockDealConverter{} + transformer = deal.DealTransformer{ + Repository: &dealRepository, + Config: config, + Fetcher: &fetcher, + Converter: &converter, + } + }) + + It("gets missing headers", func() { + err := transformer.Execute() + Expect(err).NotTo(HaveOccurred()) + Expect(dealRepository.PassedStartingBlockNumber).To(Equal(config.StartingBlockNumber)) + Expect(dealRepository.PassedEndingBlockNumber).To(Equal(config.EndingBlockNumber)) + }) + + It("returns an error if fetching the missing headers fails", func() { + dealRepository.SetMissingHeadersErr(fakes.FakeError) + err := transformer.Execute() + Expect(err).To(HaveOccurred()) + }) + + It("fetches logs for each missing header", func() { + header1 := core.Header{BlockNumber: rand.Int63()} + header2 := core.Header{BlockNumber: rand.Int63()} + dealRepository.SetMissingHeaders([]core.Header{header1, header2}) + err := transformer.Execute() + Expect(err).NotTo(HaveOccurred()) + Expect(fetcher.FetchedContractAddress).To(Equal(config.ContractAddress)) + expectedTopics := [][]common.Hash{{common.HexToHash(shared.DealSignature)}} + Expect(fetcher.FetchedTopics).To(Equal(expectedTopics)) + Expect(fetcher.FetchedBlocks).To(Equal([]int64{header1.BlockNumber, header2.BlockNumber})) + }) + + It("returns an error if fetching logs fails", func() { + dealRepository.SetMissingHeaders([]core.Header{{}}) + fetcher.SetFetcherError(fakes.FakeError) + err := transformer.Execute() + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + + It("converts each eth log to a Model", func() { + dealRepository.SetMissingHeaders([]core.Header{{}}) + fetcher.SetFetchedLogs([]types.Log{test_data.DealLogNote}) + err := transformer.Execute() + Expect(err).NotTo(HaveOccurred()) + Expect(converter.LogsToConvert).To(Equal([]types.Log{test_data.DealLogNote})) + }) + + It("returns an error if converting the eth log fails", func() { + dealRepository.SetMissingHeaders([]core.Header{{}}) + fetcher.SetFetchedLogs([]types.Log{test_data.DealLogNote}) + converter.SetConverterError(fakes.FakeError) + err := transformer.Execute() + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + + It("persists each model as a Deal record", func() { + header1 := core.Header{Id: rand.Int63()} + header2 := core.Header{Id: rand.Int63()} + dealRepository.SetMissingHeaders([]core.Header{header1, header2}) + fetcher.SetFetchedLogs([]types.Log{test_data.DealLogNote}) + err := transformer.Execute() + Expect(err).NotTo(HaveOccurred()) + Expect(dealRepository.PassedDealModels).To(Equal([]deal.DealModel{test_data.DealModel, test_data.DealModel})) + Expect(dealRepository.PassedHeaderIDs).To(Equal([]int64{header1.Id, header2.Id})) + }) + + It("returns an error if persisting deal record fails", func() { + dealRepository.SetMissingHeaders([]core.Header{{}}) + dealRepository.SetCreateError(fakes.FakeError) + fetcher.SetFetchedLogs([]types.Log{test_data.DealLogNote}) + err := transformer.Execute() + Expect(err).To(HaveOccurred()) + }) +}) diff --git a/pkg/transformers/dent/converter.go b/pkg/transformers/dent/converter.go index ad487d48..8b7d4acb 100644 --- a/pkg/transformers/dent/converter.go +++ b/pkg/transformers/dent/converter.go @@ -24,7 +24,7 @@ import ( ) type Converter interface { - Convert(contractAddress string, contractAbi string, ethLog types.Log) (DentModel, error) + ToModel(ethLog types.Log) (DentModel, error) } type DentConverter struct{} @@ -33,7 +33,7 @@ func NewDentConverter() DentConverter { return DentConverter{} } -func (c DentConverter) Convert(contractAddress, contractAbi string, ethLog types.Log) (DentModel, error) { +func (c DentConverter) ToModel(ethLog types.Log) (DentModel, error) { err := validateLog(ethLog) if err != nil { return DentModel{}, err diff --git a/pkg/transformers/dent/converter_test.go b/pkg/transformers/dent/converter_test.go index fbecd4f3..8e470188 100644 --- a/pkg/transformers/dent/converter_test.go +++ b/pkg/transformers/dent/converter_test.go @@ -20,7 +20,6 @@ import ( . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/transformers/dent" - "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" ) @@ -32,7 +31,7 @@ var _ = Describe("Dent Converter", func() { }) It("converts an eth log to a db model", func() { - model, err := converter.Convert(shared.FlipperContractAddress, shared.FlipperABI, test_data.DentLog) + model, err := converter.ToModel(test_data.DentLog) Expect(err).NotTo(HaveOccurred()) Expect(model).To(Equal(test_data.DentModel)) @@ -41,7 +40,7 @@ var _ = Describe("Dent Converter", func() { It("returns an error if the expected amount of topics aren't in the log", func() { invalidLog := test_data.DentLog invalidLog.Topics = []common.Hash{} - model, err := converter.Convert(shared.FlipperContractAddress, shared.FlipperABI, invalidLog) + model, err := converter.ToModel(invalidLog) Expect(err).To(HaveOccurred()) Expect(err).To(MatchError("dent log does not contain expected topics")) Expect(model).To(Equal(dent.DentModel{})) @@ -50,7 +49,7 @@ var _ = Describe("Dent Converter", func() { It("returns an error if the log data is empty", func() { emptyDataLog := test_data.DentLog emptyDataLog.Data = []byte{} - model, err := converter.Convert(shared.FlipperContractAddress, shared.FlipperABI, emptyDataLog) + model, err := converter.ToModel(emptyDataLog) Expect(err).To(HaveOccurred()) Expect(err).To(MatchError("dent log data is empty")) Expect(model).To(Equal(dent.DentModel{})) diff --git a/pkg/transformers/dent/transformer.go b/pkg/transformers/dent/transformer.go index 035f5eb1..c93905b9 100644 --- a/pkg/transformers/dent/transformer.go +++ b/pkg/transformers/dent/transformer.go @@ -60,7 +60,7 @@ func (t DentTransformer) Execute() error { } for _, ethLog := range ethLogs { - model, err := t.Converter.Convert(config.ContractAddress, config.ContractAbi, ethLog) + model, err := t.Converter.ToModel(ethLog) if err != nil { log.Println("Error converting dent log", err) diff --git a/pkg/transformers/dent/transformer_test.go b/pkg/transformers/dent/transformer_test.go index 01969c17..90146894 100644 --- a/pkg/transformers/dent/transformer_test.go +++ b/pkg/transformers/dent/transformer_test.go @@ -93,8 +93,6 @@ var _ = Describe("DentTransformer", func() { err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(converter.PassedContractAddress).To(Equal(config.ContractAddress)) - Expect(converter.PassedContractAbi).To(Equal(config.ContractAbi)) Expect(converter.LogsToConvert).To(Equal([]types.Log{test_data.DentLog})) }) diff --git a/pkg/transformers/shared/constants.go b/pkg/transformers/shared/constants.go index 803a7136..7f0fba57 100644 --- a/pkg/transformers/shared/constants.go +++ b/pkg/transformers/shared/constants.go @@ -36,6 +36,7 @@ var ( //TODO: get pit and drip file method signatures directly from the ABI biteMethod = GetSolidityMethodSignature(CatABI, "Bite") + dealMethod = GetSolidityMethodSignature(FlipperABI, "deal") dentMethod = GetSolidityMethodSignature(FlipperABI, "dent") dripFileIlkMethod = "file(bytes32,bytes32,uint256)" dripFileRepoMethod = GetSolidityMethodSignature(DripABI, "file") @@ -50,6 +51,7 @@ var ( vatInitMethod = GetSolidityMethodSignature(VatABI, "init") BiteSignature = GetEventSignature(biteMethod) + DealSignature = GetLogNoteSignature(dealMethod) DentFunctionSignature = GetLogNoteSignature(dentMethod) DripFileIlkSignature = GetLogNoteSignature(dripFileIlkMethod) DripFileRepoSignature = GetLogNoteSignature(dripFileRepoMethod) diff --git a/pkg/transformers/shared/event_signature_generator_test.go b/pkg/transformers/shared/event_signature_generator_test.go index 251de8d3..85ce065d 100644 --- a/pkg/transformers/shared/event_signature_generator_test.go +++ b/pkg/transformers/shared/event_signature_generator_test.go @@ -112,6 +112,12 @@ var _ = Describe("Event signature generator", func() { Expect(expected).To(Equal(actual)) }) + It("gets the flip deal method signature", func() { + expected := "deal(uint256)" + actual := shared.GetSolidityMethodSignature(shared.FlipperABI, "deal") + + Expect(expected).To(Equal(actual)) + }) }) Describe("it handles events", func() { diff --git a/pkg/transformers/test_data/deal.go b/pkg/transformers/test_data/deal.go new file mode 100644 index 00000000..2e746939 --- /dev/null +++ b/pkg/transformers/test_data/deal.go @@ -0,0 +1,48 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test_data + +import ( + "encoding/json" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/transformers/deal" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" +) + +var DealLogNote = types.Log{ + Address: common.HexToAddress(shared.FlipperContractAddress), + Topics: []common.Hash{ + common.HexToHash("0xc959c42b00000000000000000000000000000000000000000000000000000000"), + common.HexToHash("0x00000000000000000000000064d922894153be9eef7b7218dc565d1d0ce2a092"), + common.HexToHash("0x000000000000000000000000000000000000000000000000000000000000007b"), + common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000"), + }, + Data: hexutil.MustDecode("0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000000000000000000000000000024c959c42b000000000000000000000000000000000000000000000000000000000000007b"), + BlockNumber: 16, + TxHash: common.HexToHash("0xc6ff19de9299e5b290ba2d52fdb4662360ca86376613d78ee546244866a0be2d"), + TxIndex: 74, + BlockHash: common.HexToHash("0x6454844075164a1d264c86d2a2c31ac1b64eb2f4ebdbbaeb4d44388fdf74470b"), + Index: 0, + Removed: false, +} +var dealRawJson, _ = json.Marshal(DealLogNote) + +var DealModel = deal.DealModel{ + BidId: "123", + TransactionIndex: 74, + Raw: dealRawJson, +} diff --git a/pkg/transformers/test_data/mocks/deal/converter.go b/pkg/transformers/test_data/mocks/deal/converter.go new file mode 100644 index 00000000..083da8ee --- /dev/null +++ b/pkg/transformers/test_data/mocks/deal/converter.go @@ -0,0 +1,38 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package deal + +import ( + "github.com/ethereum/go-ethereum/core/types" + + "github.com/vulcanize/vulcanizedb/pkg/transformers/deal" + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" +) + +type MockDealConverter struct { + ConverterContract string + ConverterAbi string + LogsToConvert []types.Log + ConverterError error +} + +func (c *MockDealConverter) ToModel(ethLog types.Log) (deal.DealModel, error) { + c.LogsToConvert = append(c.LogsToConvert, ethLog) + return test_data.DealModel, c.ConverterError +} + +func (c *MockDealConverter) SetConverterError(err error) { + c.ConverterError = err +} diff --git a/pkg/transformers/test_data/mocks/deal/repository.go b/pkg/transformers/test_data/mocks/deal/repository.go new file mode 100644 index 00000000..b99d3628 --- /dev/null +++ b/pkg/transformers/test_data/mocks/deal/repository.go @@ -0,0 +1,54 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package deal + +import ( + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/transformers/deal" +) + +type MockDealRepository struct { + createError error + PassedEndingBlockNumber int64 + PassedHeaderIDs []int64 + PassedStartingBlockNumber int64 + PassedDealModels []deal.DealModel + missingHeaders []core.Header + missingHeadersErr error +} + +func (repository *MockDealRepository) Create(headerId int64, deal deal.DealModel) error { + repository.PassedHeaderIDs = append(repository.PassedHeaderIDs, headerId) + repository.PassedDealModels = append(repository.PassedDealModels, deal) + return repository.createError +} + +func (repository *MockDealRepository) SetCreateError(err error) { + repository.createError = err +} + +func (repository *MockDealRepository) SetMissingHeadersErr(err error) { + repository.missingHeadersErr = err +} + +func (repository *MockDealRepository) SetMissingHeaders(headers []core.Header) { + repository.missingHeaders = headers +} + +func (repository *MockDealRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + repository.PassedStartingBlockNumber = startingBlockNumber + repository.PassedEndingBlockNumber = endingBlockNumber + return repository.missingHeaders, repository.missingHeadersErr +} diff --git a/pkg/transformers/test_data/mocks/dent/converter.go b/pkg/transformers/test_data/mocks/dent/converter.go index 648946cd..22701ee9 100644 --- a/pkg/transformers/test_data/mocks/dent/converter.go +++ b/pkg/transformers/test_data/mocks/dent/converter.go @@ -28,9 +28,7 @@ type MockDentConverter struct { LogsToConvert []types.Log } -func (c *MockDentConverter) Convert(contractAddress string, contractAbi string, ethLog types.Log) (dent.DentModel, error) { - c.PassedContractAddress = contractAddress - c.PassedContractAbi = contractAbi +func (c *MockDentConverter) ToModel(ethLog types.Log) (dent.DentModel, error) { c.LogsToConvert = append(c.LogsToConvert, ethLog) return test_data.DentModel, c.converterError } diff --git a/pkg/transformers/transformers.go b/pkg/transformers/transformers.go index d65d843b..96d1df83 100644 --- a/pkg/transformers/transformers.go +++ b/pkg/transformers/transformers.go @@ -16,6 +16,7 @@ package transformers import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/bite" + "github.com/vulcanize/vulcanizedb/pkg/transformers/deal" "github.com/vulcanize/vulcanizedb/pkg/transformers/dent" "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file" ilk2 "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/ilk" @@ -34,6 +35,7 @@ import ( func TransformerInitializers() []shared.TransformerInitializer { biteTransformerInitializer := bite.BiteTransformerInitializer{Config: bite.BiteConfig} + dealTransformerInitializer := deal.DealTransformerInitializer{Config: deal.Config} dentTransformerInitializer := dent.DentTransformerInitializer{Config: dent.DentConfig} flipKickTransformerInitializer := flip_kick.FlipKickTransformerInitializer{Config: flip_kick.FlipKickConfig} frobTransformerInitializer := frob.FrobTransformerInitializer{Config: frob.FrobConfig} @@ -46,11 +48,11 @@ func TransformerInitializers() []shared.TransformerInitializer { pitFileStabilityFeeTransformerInitializer := stability_fee.PitFileStabilityFeeTransformerInitializer{Config: pitFileConfig} priceFeedTransformerInitializer := price_feeds.PriceFeedTransformerInitializer{Config: price_feeds.PriceFeedConfig} tendTransformerInitializer := tend.TendTransformerInitializer{Config: tend.TendConfig} - vatInitConfig := vat_init.VatInitConfig - vatInitTransformerInitializer := vat_init.VatInitTransformerInitializer{Config: vatInitConfig} + vatInitTransformerInitializer := vat_init.VatInitTransformerInitializer{Config: vat_init.VatInitConfig} return []shared.TransformerInitializer{ biteTransformerInitializer.NewBiteTransformer, + dealTransformerInitializer.NewDealTransformer, dentTransformerInitializer.NewDentTransformer, dripFileIlkTransformerInitializer.NewDripFileIlkTransformer, dripFileRepoTransformerInitializer.NewDripFileRepoTransformer,