diff --git a/cmd/continuousLogSync.go b/cmd/continuousLogSync.go index 7d9d0e5b..9497d30b 100644 --- a/cmd/continuousLogSync.go +++ b/cmd/continuousLogSync.go @@ -100,6 +100,7 @@ func buildTransformerInitializerMap() map[string]shared2.TransformerInitializer transformerInitializerMap["priceFeed"] = transformers.PriceFeedTransformerInitializer transformerInitializerMap["tend"] = transformers.TendTransformerInitializer transformerInitializerMap["vatInit"] = transformers.VatInitTransformerInitializer + transformerInitializerMap["vatToll"] = transformers.VatTollTransformerInitializer return transformerInitializerMap } diff --git a/db/migrations/1538679495_create_vat_toll.down.sql b/db/migrations/1538679495_create_vat_toll.down.sql new file mode 100644 index 00000000..5c291a92 --- /dev/null +++ b/db/migrations/1538679495_create_vat_toll.down.sql @@ -0,0 +1,3 @@ +DROP TABLE maker.vat_toll; +ALTER TABLE public.checked_headers + DROP COLUMN vat_toll_checked; \ No newline at end of file diff --git a/db/migrations/1538679495_create_vat_toll.up.sql b/db/migrations/1538679495_create_vat_toll.up.sql new file mode 100644 index 00000000..a1d4c87c --- /dev/null +++ b/db/migrations/1538679495_create_vat_toll.up.sql @@ -0,0 +1,13 @@ +CREATE TABLE maker.vat_toll ( + id SERIAL PRIMARY KEY, + header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE, + ilk TEXT, + urn TEXT, + take NUMERIC, + tx_idx INTEGER NOT NULL, + raw_log JSONB, + UNIQUE (header_id, tx_idx) +); + +ALTER TABLE public.checked_headers + ADD COLUMN vat_toll_checked BOOLEAN NOT NULL DEFAULT FALSE; \ No newline at end of file diff --git a/db/schema.sql b/db/schema.sql index 0081a856..5d950abf 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -730,6 +730,41 @@ CREATE SEQUENCE maker.vat_init_id_seq ALTER SEQUENCE maker.vat_init_id_seq OWNED BY maker.vat_init.id; +-- +-- Name: vat_toll; Type: TABLE; Schema: maker; Owner: - +-- + +CREATE TABLE maker.vat_toll ( + id integer NOT NULL, + header_id integer NOT NULL, + ilk text, + urn text, + take numeric, + tx_idx integer NOT NULL, + raw_log jsonb +); + + +-- +-- Name: vat_toll_id_seq; Type: SEQUENCE; Schema: maker; Owner: - +-- + +CREATE SEQUENCE maker.vat_toll_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: vat_toll_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: - +-- + +ALTER SEQUENCE maker.vat_toll_id_seq OWNED BY maker.vat_toll.id; + + -- -- Name: logs; Type: TABLE; Schema: public; Owner: - -- @@ -830,7 +865,8 @@ CREATE TABLE public.checked_headers ( pit_file_debt_ceiling_checked boolean DEFAULT false NOT NULL, pit_file_ilk_checked boolean DEFAULT false NOT NULL, pit_file_stability_fee_checked boolean DEFAULT false NOT NULL, - vat_init_checked boolean DEFAULT false NOT NULL + vat_init_checked boolean DEFAULT false NOT NULL, + vat_toll_checked boolean DEFAULT false NOT NULL ); @@ -1284,6 +1320,13 @@ ALTER TABLE ONLY maker.tend ALTER COLUMN id SET DEFAULT nextval('maker.tend_id_s ALTER TABLE ONLY maker.vat_init ALTER COLUMN id SET DEFAULT nextval('maker.vat_init_id_seq'::regclass); +-- +-- Name: vat_toll id; Type: DEFAULT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.vat_toll ALTER COLUMN id SET DEFAULT nextval('maker.vat_toll_id_seq'::regclass); + + -- -- Name: blocks id; Type: DEFAULT; Schema: public; Owner: - -- @@ -1674,6 +1717,22 @@ ALTER TABLE ONLY maker.vat_init ADD CONSTRAINT vat_init_pkey PRIMARY KEY (id); +-- +-- Name: vat_toll vat_toll_header_id_tx_idx_key; Type: CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.vat_toll + ADD CONSTRAINT vat_toll_header_id_tx_idx_key UNIQUE (header_id, tx_idx); + + +-- +-- Name: vat_toll vat_toll_pkey; Type: CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.vat_toll + ADD CONSTRAINT vat_toll_pkey PRIMARY KEY (id); + + -- -- Name: blocks blocks_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -1980,6 +2039,14 @@ ALTER TABLE ONLY maker.vat_init ADD CONSTRAINT vat_init_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE; +-- +-- Name: vat_toll vat_toll_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.vat_toll + ADD CONSTRAINT vat_toll_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE; + + -- -- Name: transactions blocks_fk; Type: FK CONSTRAINT; Schema: public; Owner: - -- diff --git a/pkg/transformers/shared/constants.go b/pkg/transformers/shared/constants.go index fd3f03d3..2e411204 100644 --- a/pkg/transformers/shared/constants.go +++ b/pkg/transformers/shared/constants.go @@ -56,6 +56,7 @@ var ( pitFileStabilityFeeMethod = GetSolidityMethodSignature(PitABI, "file") tendMethod = GetSolidityMethodSignature(FlipperABI, "tend") vatInitMethod = GetSolidityMethodSignature(VatABI, "init") + vatTollMethod = GetSolidityMethodSignature(VatABI, "toll") BiteSignature = GetEventSignature(biteMethod) DealSignature = GetLogNoteSignature(dealMethod) @@ -76,4 +77,5 @@ var ( PitFileStabilityFeeSignature = GetLogNoteSignature(pitFileStabilityFeeMethod) TendFunctionSignature = GetLogNoteSignature(tendMethod) VatInitSignature = GetLogNoteSignature(vatInitMethod) + VatTollSignature = GetLogNoteSignature(vatTollMethod) ) diff --git a/pkg/transformers/test_data/mocks/vat_toll/converter.go b/pkg/transformers/test_data/mocks/vat_toll/converter.go new file mode 100644 index 00000000..5d1c2604 --- /dev/null +++ b/pkg/transformers/test_data/mocks/vat_toll/converter.go @@ -0,0 +1,21 @@ +package vat_toll + +import ( + "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_toll" +) + +type MockVatTollConverter struct { + err error + PassedLogs []types.Log +} + +func (converter *MockVatTollConverter) ToModels(ethLogs []types.Log) ([]vat_toll.VatTollModel, error) { + converter.PassedLogs = ethLogs + return []vat_toll.VatTollModel{test_data.VatTollModel}, converter.err +} + +func (converter *MockVatTollConverter) SetConverterError(e error) { + converter.err = e +} diff --git a/pkg/transformers/test_data/mocks/vat_toll/repository.go b/pkg/transformers/test_data/mocks/vat_toll/repository.go new file mode 100644 index 00000000..094f45a9 --- /dev/null +++ b/pkg/transformers/test_data/mocks/vat_toll/repository.go @@ -0,0 +1,57 @@ +package vat_toll + +import ( + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_toll" +) + +type MockVatTollRepository struct { + createErr error + markHeaderCheckedErr error + markHeaderCheckedPassedHeaderID int64 + missingHeadersErr error + missingHeaders []core.Header + PassedStartingBlockNumber int64 + PassedEndingBlockNumber int64 + PassedHeaderID int64 + PassedModels []vat_toll.VatTollModel +} + +func (repository *MockVatTollRepository) Create(headerID int64, models []vat_toll.VatTollModel) error { + repository.PassedHeaderID = headerID + repository.PassedModels = models + return repository.createErr +} + +func (repository *MockVatTollRepository) MarkHeaderChecked(headerID int64) error { + repository.markHeaderCheckedPassedHeaderID = headerID + return repository.markHeaderCheckedErr +} + +func (repository *MockVatTollRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + repository.PassedStartingBlockNumber = startingBlockNumber + repository.PassedEndingBlockNumber = endingBlockNumber + return repository.missingHeaders, repository.missingHeadersErr +} + +func (repository *MockVatTollRepository) SetCreateError(e error) { + repository.createErr = e +} + +func (repository *MockVatTollRepository) SetMarkHeaderCheckedErr(e error) { + repository.markHeaderCheckedErr = e +} + +func (repository *MockVatTollRepository) SetMissingHeadersErr(e error) { + repository.missingHeadersErr = e +} + +func (repository *MockVatTollRepository) SetMissingHeaders(headers []core.Header) { + repository.missingHeaders = headers +} + +func (repository *MockVatTollRepository) AssertMarkHeaderCheckedCalledWith(i int64) { + Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i)) +} diff --git a/pkg/transformers/test_data/vat_toll.go b/pkg/transformers/test_data/vat_toll.go new file mode 100644 index 00000000..bc66d62b --- /dev/null +++ b/pkg/transformers/test_data/vat_toll.go @@ -0,0 +1,36 @@ +package test_data + +import ( + "encoding/json" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_toll" + "math/big" +) + +var EthVatTollLog = types.Log{ + Address: common.HexToAddress("0x239E6f0AB02713f1F8AA90ebeDeD9FC66Dc96CD6"), + Topics: []common.Hash{ + common.HexToHash("0x09b7a0b500000000000000000000000000000000000000000000000000000000"), + common.HexToHash("0x66616b6520696c6b000000000000000000000000000000000000000000000000"), + common.HexToHash("0xa3e37186e017747dba34042e83e3f76ad3cce9b0000000000000000000000000"), + common.HexToHash("0x00000000000000000000000000000000000000000000000000000000075bcd15"), + }, + Data: hexutil.MustDecode("0x0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000006409b7a0b566616b6520696c6b000000000000000000000000000000000000000000000000a3e37186e017747dba34042e83e3f76ad3cce9b000000000000000000000000000000000000000000000000000000000000000000000000000000000075bcd15"), + BlockNumber: 21, + TxHash: common.HexToHash("0x0d59cb158b033ffdfb9a021d1e80bfbbcd99594c62c501897ccee446bcd33828"), + TxIndex: 2, + BlockHash: common.HexToHash("0xdbfda0ecb4ac6267c50be21db97874dc1203c22033e19733a6f67ca00b51dfc5"), + Index: 4, + Removed: false, +} + +var rawVatTollLog, _ = json.Marshal(EthVatTollLog) +var VatTollModel = vat_toll.VatTollModel{ + Ilk: "fake ilk", + Urn: "0xA3E37186E017747DbA34042e83e3F76Ad3CcE9b0", + Take: big.NewInt(123456789).String(), + TransactionIndex: EthVatTollLog.TxIndex, + Raw: rawVatTollLog, +} diff --git a/pkg/transformers/transformers.go b/pkg/transformers/transformers.go index 6aa22de7..8d823834 100644 --- a/pkg/transformers/transformers.go +++ b/pkg/transformers/transformers.go @@ -38,6 +38,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/tend" "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_init" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_toll" ) var ( @@ -63,6 +64,7 @@ var ( PriceFeedTransformerInitializer = price_feeds.PriceFeedTransformerInitializer{Config: price_feeds.PriceFeedConfig}.NewPriceFeedTransformer TendTransformerInitializer = tend.TendTransformerInitializer{Config: tend.TendConfig}.NewTendTransformer VatInitTransformerInitializer = vat_init.VatInitTransformerInitializer{Config: vat_init.VatInitConfig}.NewVatInitTransformer + VatTollTransformerInitializer = vat_toll.VatTollTransformerInitializer{Config: vat_toll.VatTollConfig}.NewVatTollTransformer ) func TransformerInitializers() []shared.TransformerInitializer { @@ -86,5 +88,6 @@ func TransformerInitializers() []shared.TransformerInitializer { PriceFeedTransformerInitializer, TendTransformerInitializer, VatInitTransformerInitializer, + VatTollTransformerInitializer, } } diff --git a/pkg/transformers/vat_toll/config.go b/pkg/transformers/vat_toll/config.go new file mode 100644 index 00000000..dcace67e --- /dev/null +++ b/pkg/transformers/vat_toll/config.go @@ -0,0 +1,11 @@ +package vat_toll + +import "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + +var VatTollConfig = shared.TransformerConfig{ + ContractAddresses: []string{shared.VatContractAddress}, + ContractAbi: shared.VatABI, + Topics: []string{shared.VatTollSignature}, + StartingBlockNumber: 0, + EndingBlockNumber: 10000000, +} diff --git a/pkg/transformers/vat_toll/converter.go b/pkg/transformers/vat_toll/converter.go new file mode 100644 index 00000000..6998b423 --- /dev/null +++ b/pkg/transformers/vat_toll/converter.go @@ -0,0 +1,51 @@ +package vat_toll + +import ( + "bytes" + "encoding/json" + "errors" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" +) + +type Converter interface { + ToModels(ethLogs []types.Log) ([]VatTollModel, error) +} + +type VatTollConverter struct{} + +func (VatTollConverter) ToModels(ethLogs []types.Log) ([]VatTollModel, error) { + var models []VatTollModel + for _, ethLog := range ethLogs { + err := verifyLog(ethLog) + if err != nil { + return nil, err + } + ilk := string(bytes.Trim(ethLog.Topics[1].Bytes(), "\x00")) + urn := common.BytesToAddress(ethLog.Topics[2].Bytes()[:common.AddressLength]) + take := ethLog.Topics[3].Big() + + raw, err := json.Marshal(ethLog) + if err != nil { + return nil, err + } + model := VatTollModel{ + Ilk: ilk, + Urn: urn.String(), + Take: take.String(), + TransactionIndex: ethLog.TxIndex, + Raw: raw, + } + models = append(models, model) + } + return models, nil +} + +func verifyLog(log types.Log) error { + numTopicInValidLog := 4 + if len(log.Topics) < numTopicInValidLog { + return errors.New("log missing topics") + } + return nil +} diff --git a/pkg/transformers/vat_toll/converter_test.go b/pkg/transformers/vat_toll/converter_test.go new file mode 100644 index 00000000..94b049f3 --- /dev/null +++ b/pkg/transformers/vat_toll/converter_test.go @@ -0,0 +1,33 @@ +package vat_toll_test + +import ( + "github.com/ethereum/go-ethereum/core/types" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_toll" +) + +var _ = Describe("Vat toll converter", func() { + It("returns err if log is missing topics", func() { + converter := vat_toll.VatTollConverter{} + badLog := types.Log{ + Data: []byte{1, 1, 1, 1, 1}, + } + + _, err := converter.ToModels([]types.Log{badLog}) + + Expect(err).To(HaveOccurred()) + }) + + It("converts a log to an model", func() { + converter := vat_toll.VatTollConverter{} + + models, err := converter.ToModels([]types.Log{test_data.EthVatTollLog}) + + Expect(err).NotTo(HaveOccurred()) + Expect(len(models)).To(Equal(1)) + Expect(models[0]).To(Equal(test_data.VatTollModel)) + }) +}) diff --git a/pkg/transformers/vat_toll/model.go b/pkg/transformers/vat_toll/model.go new file mode 100644 index 00000000..a0660ad1 --- /dev/null +++ b/pkg/transformers/vat_toll/model.go @@ -0,0 +1,9 @@ +package vat_toll + +type VatTollModel struct { + Ilk string + Urn string + Take string + TransactionIndex uint `db:"tx_idx"` + Raw []byte `db:"raw_log"` +} diff --git a/pkg/transformers/vat_toll/repository.go b/pkg/transformers/vat_toll/repository.go new file mode 100644 index 00000000..2057c5e6 --- /dev/null +++ b/pkg/transformers/vat_toll/repository.go @@ -0,0 +1,74 @@ +package vat_toll + +import ( + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" +) + +type Repository interface { + Create(headerID int64, models []VatTollModel) error + MarkHeaderChecked(headerID int64) error + MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) +} + +type VatTollRepository struct { + db *postgres.DB +} + +func NewVatTollRepository(db *postgres.DB) VatTollRepository { + return VatTollRepository{ + db: db, + } +} + +func (repository VatTollRepository) Create(headerID int64, models []VatTollModel) error { + tx, err := repository.db.Begin() + if err != nil { + return err + } + for _, model := range models { + _, err = tx.Exec( + `INSERT into maker.vat_toll (header_id, ilk, urn, take, tx_idx, raw_log) + VALUES($1, $2, $3, $4::NUMERIC, $5, $6)`, + headerID, model.Ilk, model.Urn, model.Take, model.TransactionIndex, model.Raw, + ) + if err != nil { + tx.Rollback() + return err + } + } + _, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, vat_toll_checked) + VALUES ($1, $2) + ON CONFLICT (header_id) DO + UPDATE SET vat_toll_checked = $2`, headerID, true) + if err != nil { + tx.Rollback() + return err + } + return tx.Commit() +} + +func (repository VatTollRepository) MarkHeaderChecked(headerID int64) error { + _, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, vat_toll_checked) + VALUES ($1, $2) + ON CONFLICT (header_id) DO + UPDATE SET vat_toll_checked = $2`, headerID, true) + return err +} + +func (repository VatTollRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + var result []core.Header + err := repository.db.Select( + &result, + `SELECT headers.id, headers.block_number FROM headers + LEFT JOIN checked_headers on headers.id = header_id + WHERE (header_id ISNULL OR vat_toll_checked IS FALSE) + AND headers.block_number >= $1 + AND headers.block_number <= $2 + AND headers.eth_node_fingerprint = $3`, + startingBlockNumber, + endingBlockNumber, + repository.db.Node.ID, + ) + return result, err +} diff --git a/pkg/transformers/vat_toll/repository_test.go b/pkg/transformers/vat_toll/repository_test.go new file mode 100644 index 00000000..241ff475 --- /dev/null +++ b/pkg/transformers/vat_toll/repository_test.go @@ -0,0 +1,206 @@ +package vat_toll_test + +import ( + "database/sql" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_toll" + "github.com/vulcanize/vulcanizedb/test_config" +) + +var _ = Describe("Vat toll repository", func() { + Describe("Create", func() { + var ( + db *postgres.DB + vatTollRepository vat_toll.Repository + err error + headerID int64 + ) + + BeforeEach(func() { + db = test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{}) + Expect(err).NotTo(HaveOccurred()) + vatTollRepository = vat_toll.NewVatTollRepository(db) + }) + + It("adds a vat toll event", func() { + err = vatTollRepository.Create(headerID, []vat_toll.VatTollModel{test_data.VatTollModel}) + + Expect(err).NotTo(HaveOccurred()) + var dbVatToll vat_toll.VatTollModel + err = db.Get(&dbVatToll, `SELECT ilk, urn, take, tx_idx, raw_log FROM maker.vat_toll WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(dbVatToll.Ilk).To(Equal(test_data.VatTollModel.Ilk)) + Expect(dbVatToll.Urn).To(Equal(test_data.VatTollModel.Urn)) + Expect(dbVatToll.Take).To(Equal(test_data.VatTollModel.Take)) + Expect(dbVatToll.TransactionIndex).To(Equal(test_data.VatTollModel.TransactionIndex)) + Expect(dbVatToll.Raw).To(MatchJSON(test_data.VatTollModel.Raw)) + }) + + It("marks header as checked for logs", func() { + err = vatTollRepository.Create(headerID, []vat_toll.VatTollModel{test_data.VatTollModel}) + + Expect(err).NotTo(HaveOccurred()) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT vat_toll_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + + It("does not duplicate vat toll events", func() { + err = vatTollRepository.Create(headerID, []vat_toll.VatTollModel{test_data.VatTollModel}) + Expect(err).NotTo(HaveOccurred()) + + err = vatTollRepository.Create(headerID, []vat_toll.VatTollModel{test_data.VatTollModel}) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint")) + }) + + It("removes vat toll if corresponding header is deleted", func() { + err = vatTollRepository.Create(headerID, []vat_toll.VatTollModel{test_data.VatTollModel}) + Expect(err).NotTo(HaveOccurred()) + + _, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID) + + Expect(err).NotTo(HaveOccurred()) + var dbVatToll vat_toll.VatTollModel + err = db.Get(&dbVatToll, `SELECT ilk, urn, take, tx_idx, raw_log FROM maker.vat_toll WHERE header_id = $1`, headerID) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(sql.ErrNoRows)) + }) + }) + + Describe("MarkHeaderChecked", func() { + var ( + db *postgres.DB + vatTollRepository vat_toll.Repository + err error + headerID int64 + ) + + BeforeEach(func() { + db = test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{}) + Expect(err).NotTo(HaveOccurred()) + vatTollRepository = vat_toll.NewVatTollRepository(db) + }) + + It("creates a row for a new headerID", func() { + err = vatTollRepository.MarkHeaderChecked(headerID) + + Expect(err).NotTo(HaveOccurred()) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT vat_toll_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + + It("updates row when headerID already exists", func() { + _, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID) + + err = vatTollRepository.MarkHeaderChecked(headerID) + + Expect(err).NotTo(HaveOccurred()) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT vat_toll_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + }) + + Describe("MissingHeaders", func() { + It("returns headers that haven't been checked", func() { + db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + startingBlockNumber := int64(1) + pitFileBlockNumber := int64(2) + endingBlockNumber := int64(3) + blockNumbers := []int64{startingBlockNumber, pitFileBlockNumber, endingBlockNumber, endingBlockNumber + 1} + var headerIDs []int64 + for _, n := range blockNumbers { + headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + headerIDs = append(headerIDs, headerID) + Expect(err).NotTo(HaveOccurred()) + } + pitFileRepository := vat_toll.NewVatTollRepository(db) + err := pitFileRepository.MarkHeaderChecked(headerIDs[1]) + Expect(err).NotTo(HaveOccurred()) + + headers, err := pitFileRepository.MissingHeaders(startingBlockNumber, endingBlockNumber) + + Expect(err).NotTo(HaveOccurred()) + Expect(len(headers)).To(Equal(2)) + Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber))) + Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber))) + }) + + It("only treats headers as checked if vat toll logs have been checked", func() { + db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + startingBlockNumber := int64(1) + vatTolldBlockNumber := int64(2) + endingBlockNumber := int64(3) + blockNumbers := []int64{startingBlockNumber, vatTolldBlockNumber, endingBlockNumber, endingBlockNumber + 1} + var headerIDs []int64 + for _, n := range blockNumbers { + headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + headerIDs = append(headerIDs, headerID) + Expect(err).NotTo(HaveOccurred()) + } + vatTollRepository := vat_toll.NewVatTollRepository(db) + _, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1]) + Expect(err).NotTo(HaveOccurred()) + + headers, err := vatTollRepository.MissingHeaders(startingBlockNumber, endingBlockNumber) + + Expect(err).NotTo(HaveOccurred()) + Expect(len(headers)).To(Equal(3)) + Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatTolldBlockNumber))) + Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatTolldBlockNumber))) + Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatTolldBlockNumber))) + }) + + It("only returns headers associated with the current node", func() { + db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + blockNumbers := []int64{1, 2, 3} + headerRepository := repositories.NewHeaderRepository(db) + dbTwo := test_config.NewTestDB(core.Node{ID: "second"}) + headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo) + var headerIDs []int64 + for _, n := range blockNumbers { + headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + Expect(err).NotTo(HaveOccurred()) + headerIDs = append(headerIDs, headerID) + _, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + Expect(err).NotTo(HaveOccurred()) + } + pitFileRepository := vat_toll.NewVatTollRepository(db) + pitFileRepositoryTwo := vat_toll.NewVatTollRepository(dbTwo) + err := pitFileRepository.MarkHeaderChecked(headerIDs[0]) + Expect(err).NotTo(HaveOccurred()) + + nodeOneMissingHeaders, err := pitFileRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) + Expect(err).NotTo(HaveOccurred()) + Expect(len(nodeOneMissingHeaders)).To(Equal(len(blockNumbers) - 1)) + + nodeTwoMissingHeaders, err := pitFileRepositoryTwo.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) + Expect(err).NotTo(HaveOccurred()) + Expect(len(nodeTwoMissingHeaders)).To(Equal(len(blockNumbers))) + }) + }) +}) diff --git a/pkg/transformers/vat_toll/transformer.go b/pkg/transformers/vat_toll/transformer.go new file mode 100644 index 00000000..454e143a --- /dev/null +++ b/pkg/transformers/vat_toll/transformer.go @@ -0,0 +1,62 @@ +package vat_toll + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "log" +) + +type VatTollTransformer struct { + Config shared.TransformerConfig + Converter Converter + Fetcher shared.LogFetcher + Repository Repository +} + +type VatTollTransformerInitializer struct { + Config shared.TransformerConfig +} + +func (initializer VatTollTransformerInitializer) NewVatTollTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer { + converter := VatTollConverter{} + fetcher := shared.NewFetcher(blockChain) + repository := NewVatTollRepository(db) + return VatTollTransformer{ + Config: initializer.Config, + Converter: converter, + Fetcher: fetcher, + Repository: repository, + } +} + +func (transformer VatTollTransformer) Execute() error { + missingHeaders, err := transformer.Repository.MissingHeaders(transformer.Config.StartingBlockNumber, transformer.Config.EndingBlockNumber) + if err != nil { + return err + } + log.Printf("Fetching vat toll event logs for %d headers \n", len(missingHeaders)) + for _, header := range missingHeaders { + topics := [][]common.Hash{{common.HexToHash(shared.VatTollSignature)}} + matchingLogs, err := transformer.Fetcher.FetchLogs(VatTollConfig.ContractAddresses, topics, header.BlockNumber) + if err != nil { + return err + } + if len(matchingLogs) < 1 { + err = transformer.Repository.MarkHeaderChecked(header.Id) + if err != nil { + return err + } + } + models, err := transformer.Converter.ToModels(matchingLogs) + if err != nil { + return err + } + err = transformer.Repository.Create(header.Id, models) + if err != nil { + return err + } + } + return nil +} diff --git a/pkg/transformers/vat_toll/transformer_test.go b/pkg/transformers/vat_toll/transformer_test.go new file mode 100644 index 00000000..12a37f9d --- /dev/null +++ b/pkg/transformers/vat_toll/transformer_test.go @@ -0,0 +1,196 @@ +package vat_toll_test + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/fakes" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks" + vat_toll_mocks "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks/vat_toll" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_toll" +) + +var _ = Describe("Vat toll transformer", func() { + It("gets missing headers for block numbers specified in config", func() { + repository := &vat_toll_mocks.MockVatTollRepository{} + transformer := vat_toll.VatTollTransformer{ + Config: vat_toll.VatTollConfig, + Fetcher: &mocks.MockLogFetcher{}, + Converter: &vat_toll_mocks.MockVatTollConverter{}, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + Expect(repository.PassedStartingBlockNumber).To(Equal(vat_toll.VatTollConfig.StartingBlockNumber)) + Expect(repository.PassedEndingBlockNumber).To(Equal(vat_toll.VatTollConfig.EndingBlockNumber)) + }) + + It("returns error if repository returns error for missing headers", func() { + repository := &vat_toll_mocks.MockVatTollRepository{} + repository.SetMissingHeadersErr(fakes.FakeError) + transformer := vat_toll.VatTollTransformer{ + Fetcher: &mocks.MockLogFetcher{}, + Converter: &vat_toll_mocks.MockVatTollConverter{}, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + + It("fetches logs for missing headers", func() { + fetcher := &mocks.MockLogFetcher{} + repository := &vat_toll_mocks.MockVatTollRepository{} + repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}, {BlockNumber: 2}}) + transformer := vat_toll.VatTollTransformer{ + Fetcher: fetcher, + Converter: &vat_toll_mocks.MockVatTollConverter{}, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + Expect(fetcher.FetchedBlocks).To(Equal([]int64{1, 2})) + Expect(fetcher.FetchedContractAddresses).To(Equal([][]string{vat_toll.VatTollConfig.ContractAddresses, vat_toll.VatTollConfig.ContractAddresses})) + Expect(fetcher.FetchedTopics).To(Equal([][]common.Hash{{common.HexToHash(shared.VatTollSignature)}})) + }) + + It("returns error if fetcher returns error", func() { + fetcher := &mocks.MockLogFetcher{} + fetcher.SetFetcherError(fakes.FakeError) + repository := &vat_toll_mocks.MockVatTollRepository{} + repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) + transformer := vat_toll.VatTollTransformer{ + Fetcher: fetcher, + Converter: &vat_toll_mocks.MockVatTollConverter{}, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + + It("marks header checked if no logs returned", func() { + mockConverter := &vat_toll_mocks.MockVatTollConverter{} + mockRepository := &vat_toll_mocks.MockVatTollRepository{} + headerID := int64(123) + mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}}) + mockFetcher := &mocks.MockLogFetcher{} + transformer := vat_toll.VatTollTransformer{ + Converter: mockConverter, + Fetcher: mockFetcher, + Repository: mockRepository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + mockRepository.AssertMarkHeaderCheckedCalledWith(headerID) + }) + + It("returns error if marking header checked returns err", func() { + mockConverter := &vat_toll_mocks.MockVatTollConverter{} + mockRepository := &vat_toll_mocks.MockVatTollRepository{} + mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}}) + mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError) + mockFetcher := &mocks.MockLogFetcher{} + transformer := vat_toll.VatTollTransformer{ + Converter: mockConverter, + Fetcher: mockFetcher, + Repository: mockRepository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + + It("converts matching logs", func() { + converter := &vat_toll_mocks.MockVatTollConverter{} + fetcher := &mocks.MockLogFetcher{} + fetcher.SetFetchedLogs([]types.Log{test_data.EthVatTollLog}) + repository := &vat_toll_mocks.MockVatTollRepository{} + repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) + transformer := vat_toll.VatTollTransformer{ + Fetcher: fetcher, + Converter: converter, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + Expect(converter.PassedLogs).To(Equal([]types.Log{test_data.EthVatTollLog})) + }) + + It("returns error if converter returns error", func() { + converter := &vat_toll_mocks.MockVatTollConverter{} + converter.SetConverterError(fakes.FakeError) + fetcher := &mocks.MockLogFetcher{} + fetcher.SetFetchedLogs([]types.Log{test_data.EthVatTollLog}) + repository := &vat_toll_mocks.MockVatTollRepository{} + repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) + transformer := vat_toll.VatTollTransformer{ + Fetcher: fetcher, + Converter: converter, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + + It("persists vat toll model", func() { + converter := &vat_toll_mocks.MockVatTollConverter{} + fetcher := &mocks.MockLogFetcher{} + fetcher.SetFetchedLogs([]types.Log{test_data.EthVatTollLog}) + repository := &vat_toll_mocks.MockVatTollRepository{} + fakeHeader := core.Header{BlockNumber: 1, Id: 2} + repository.SetMissingHeaders([]core.Header{fakeHeader}) + transformer := vat_toll.VatTollTransformer{ + Fetcher: fetcher, + Converter: converter, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id)) + Expect(repository.PassedModels).To(Equal([]vat_toll.VatTollModel{test_data.VatTollModel})) + }) + + It("returns error if repository returns error for create", func() { + converter := &vat_toll_mocks.MockVatTollConverter{} + fetcher := &mocks.MockLogFetcher{} + fetcher.SetFetchedLogs([]types.Log{test_data.EthVatTollLog}) + repository := &vat_toll_mocks.MockVatTollRepository{} + repository.SetMissingHeaders([]core.Header{{BlockNumber: 1, Id: 2}}) + repository.SetCreateError(fakes.FakeError) + transformer := vat_toll.VatTollTransformer{ + Fetcher: fetcher, + Converter: converter, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) +}) diff --git a/pkg/transformers/vat_toll/vat_toll_suite_test.go b/pkg/transformers/vat_toll/vat_toll_suite_test.go new file mode 100644 index 00000000..861e1a01 --- /dev/null +++ b/pkg/transformers/vat_toll/vat_toll_suite_test.go @@ -0,0 +1,19 @@ +package vat_toll_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "io/ioutil" + "log" +) + +func TestVatToll(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "VatToll Suite") +} + +var _ = BeforeSuite(func() { + log.SetOutput(ioutil.Discard) +}) diff --git a/test_config/test_config.go b/test_config/test_config.go index 12717fe9..dd0b6a02 100644 --- a/test_config/test_config.go +++ b/test_config/test_config.go @@ -90,6 +90,7 @@ func CleanTestDB(db *postgres.DB) { db.MustExec("DELETE FROM maker.price_feeds") db.MustExec("DELETE FROM maker.tend") db.MustExec("DELETE FROM maker.vat_init") + db.MustExec("DELETE FROM maker.vat_toll") db.MustExec("DELETE FROM receipts") db.MustExec("DELETE FROM transactions") db.MustExec("DELETE FROM watched_contracts")