From e935f2f553d15b052afd391cac9203770ff7dde6 Mon Sep 17 00:00:00 2001 From: Rob Mulholand Date: Mon, 8 Oct 2018 16:39:57 -0500 Subject: [PATCH] Add Vat grab transformer --- cmd/continuousLogSync.go | 1 + .../1539033620_create_vat_grab.down.sql | 3 + .../1539033620_create_vat_grab.up.sql | 16 ++ db/schema.sql | 72 +++++- pkg/transformers/shared/constants.go | 2 + .../shared/event_signature_generator_test.go | 7 + .../test_data/mocks/vat_grab/converter.go | 22 ++ .../test_data/mocks/vat_grab/repository.go | 56 +++++ pkg/transformers/test_data/vat_grab.go | 39 ++++ pkg/transformers/transformers.go | 3 + pkg/transformers/vat_grab/config.go | 11 + pkg/transformers/vat_grab/converter.go | 63 ++++++ pkg/transformers/vat_grab/converter_test.go | 47 ++++ pkg/transformers/vat_grab/model.go | 12 + pkg/transformers/vat_grab/repository.go | 74 +++++++ pkg/transformers/vat_grab/repository_test.go | 205 ++++++++++++++++++ pkg/transformers/vat_grab/tranformer.go | 63 ++++++ pkg/transformers/vat_grab/transformer_test.go | 196 +++++++++++++++++ .../vat_grab/vat_grab_suite_test.go | 19 ++ test_config/test_config.go | 1 + 20 files changed, 911 insertions(+), 1 deletion(-) create mode 100644 db/migrations/1539033620_create_vat_grab.down.sql create mode 100644 db/migrations/1539033620_create_vat_grab.up.sql create mode 100644 pkg/transformers/test_data/mocks/vat_grab/converter.go create mode 100644 pkg/transformers/test_data/mocks/vat_grab/repository.go create mode 100644 pkg/transformers/test_data/vat_grab.go create mode 100644 pkg/transformers/vat_grab/config.go create mode 100644 pkg/transformers/vat_grab/converter.go create mode 100644 pkg/transformers/vat_grab/converter_test.go create mode 100644 pkg/transformers/vat_grab/model.go create mode 100644 pkg/transformers/vat_grab/repository.go create mode 100644 pkg/transformers/vat_grab/repository_test.go create mode 100644 pkg/transformers/vat_grab/tranformer.go create mode 100644 pkg/transformers/vat_grab/transformer_test.go create mode 100644 pkg/transformers/vat_grab/vat_grab_suite_test.go diff --git a/cmd/continuousLogSync.go b/cmd/continuousLogSync.go index 8f91b5df..e144b5f0 100644 --- a/cmd/continuousLogSync.go +++ b/cmd/continuousLogSync.go @@ -104,6 +104,7 @@ func buildTransformerInitializerMap() map[string]shared2.TransformerInitializer transformerInitializerMap["pitFileStabilityFee"] = transformers.PitFileStabilityFeeTransformerInitializer transformerInitializerMap["priceFeed"] = transformers.PriceFeedTransformerInitializer transformerInitializerMap["tend"] = transformers.TendTransformerInitializer + transformerInitializerMap["vatGrab"] = transformers.VatGrabTransformerInitializer transformerInitializerMap["vatInit"] = transformers.VatInitTransformerInitializer transformerInitializerMap["vatToll"] = transformers.VatTollTransformerInitializer transformerInitializerMap["vatTune"] = transformers.VatTuneTransformerInitializer diff --git a/db/migrations/1539033620_create_vat_grab.down.sql b/db/migrations/1539033620_create_vat_grab.down.sql new file mode 100644 index 00000000..eade21a1 --- /dev/null +++ b/db/migrations/1539033620_create_vat_grab.down.sql @@ -0,0 +1,3 @@ +DROP TABLE maker.vat_grab; +ALTER TABLE public.checked_headers + DROP COLUMN vat_grab_checked; \ No newline at end of file diff --git a/db/migrations/1539033620_create_vat_grab.up.sql b/db/migrations/1539033620_create_vat_grab.up.sql new file mode 100644 index 00000000..60b66d87 --- /dev/null +++ b/db/migrations/1539033620_create_vat_grab.up.sql @@ -0,0 +1,16 @@ +CREATE TABLE maker.vat_grab ( + id SERIAL PRIMARY KEY, + header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE, + ilk TEXT, + urn TEXT, + v TEXT, + w TEXT, + dink NUMERIC, + dart NUMERIC, + tx_idx INTEGER NOT NULL, + raw_log JSONB, + UNIQUE (header_id, tx_idx) +); + +ALTER TABLE public.checked_headers + ADD COLUMN vat_grab_checked BOOLEAN NOT NULL DEFAULT FALSE; \ No newline at end of file diff --git a/db/schema.sql b/db/schema.sql index 7d60455d..614b964d 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -697,6 +697,44 @@ CREATE SEQUENCE maker.tend_id_seq ALTER SEQUENCE maker.tend_id_seq OWNED BY maker.tend.id; +-- +-- Name: vat_grab; Type: TABLE; Schema: maker; Owner: - +-- + +CREATE TABLE maker.vat_grab ( + id integer NOT NULL, + header_id integer NOT NULL, + ilk text, + urn text, + v text, + w text, + dink numeric, + dart numeric, + tx_idx integer NOT NULL, + raw_log jsonb +); + + +-- +-- Name: vat_grab_id_seq; Type: SEQUENCE; Schema: maker; Owner: - +-- + +CREATE SEQUENCE maker.vat_grab_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: vat_grab_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: - +-- + +ALTER SEQUENCE maker.vat_grab_id_seq OWNED BY maker.vat_grab.id; + + -- -- Name: vat_init; Type: TABLE; Schema: maker; Owner: - -- @@ -905,7 +943,8 @@ CREATE TABLE public.checked_headers ( pit_file_stability_fee_checked boolean DEFAULT false NOT NULL, vat_init_checked boolean DEFAULT false NOT NULL, vat_toll_checked boolean DEFAULT false NOT NULL, - vat_tune_checked boolean DEFAULT false NOT NULL + vat_tune_checked boolean DEFAULT false NOT NULL, + vat_grab_checked boolean DEFAULT false NOT NULL ); @@ -1352,6 +1391,13 @@ ALTER TABLE ONLY maker.price_feeds ALTER COLUMN id SET DEFAULT nextval('maker.pr ALTER TABLE ONLY maker.tend ALTER COLUMN id SET DEFAULT nextval('maker.tend_id_seq'::regclass); +-- +-- Name: vat_grab id; Type: DEFAULT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.vat_grab ALTER COLUMN id SET DEFAULT nextval('maker.vat_grab_id_seq'::regclass); + + -- -- Name: vat_init id; Type: DEFAULT; Schema: maker; Owner: - -- @@ -1731,6 +1777,22 @@ ALTER TABLE ONLY maker.tend ADD CONSTRAINT tend_pkey PRIMARY KEY (id); +-- +-- Name: vat_grab vat_grab_header_id_tx_idx_key; Type: CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.vat_grab + ADD CONSTRAINT vat_grab_header_id_tx_idx_key UNIQUE (header_id, tx_idx); + + +-- +-- Name: vat_grab vat_grab_pkey; Type: CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.vat_grab + ADD CONSTRAINT vat_grab_pkey PRIMARY KEY (id); + + -- -- Name: vat_init vat_init_header_id_tx_idx_key; Type: CONSTRAINT; Schema: maker; Owner: - -- @@ -2077,6 +2139,14 @@ ALTER TABLE ONLY maker.tend ADD CONSTRAINT tend_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE; +-- +-- Name: vat_grab vat_grab_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.vat_grab + ADD CONSTRAINT vat_grab_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE; + + -- -- Name: vat_init vat_init_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - -- diff --git a/pkg/transformers/shared/constants.go b/pkg/transformers/shared/constants.go index f6232779..d235540d 100644 --- a/pkg/transformers/shared/constants.go +++ b/pkg/transformers/shared/constants.go @@ -55,6 +55,7 @@ var ( pitFileIlkMethod = "file(bytes32,bytes32,uint256)" pitFileStabilityFeeMethod = GetSolidityMethodSignature(PitABI, "file") tendMethod = GetSolidityMethodSignature(FlipperABI, "tend") + vatGrabMethod = GetSolidityMethodSignature(VatABI, "grab") vatInitMethod = GetSolidityMethodSignature(VatABI, "init") vatTollMethod = GetSolidityMethodSignature(VatABI, "toll") vatTuneMethod = GetSolidityMethodSignature(VatABI, "tune") @@ -77,6 +78,7 @@ var ( PitFileIlkSignature = GetLogNoteSignature(pitFileIlkMethod) PitFileStabilityFeeSignature = GetLogNoteSignature(pitFileStabilityFeeMethod) TendFunctionSignature = GetLogNoteSignature(tendMethod) + VatGrabSignature = GetLogNoteSignature(vatGrabMethod) VatInitSignature = GetLogNoteSignature(vatInitMethod) VatTollSignature = GetLogNoteSignature(vatTollMethod) VatTuneSignature = GetLogNoteSignature(vatTuneMethod) diff --git a/pkg/transformers/shared/event_signature_generator_test.go b/pkg/transformers/shared/event_signature_generator_test.go index 3bed6e80..248a84c3 100644 --- a/pkg/transformers/shared/event_signature_generator_test.go +++ b/pkg/transformers/shared/event_signature_generator_test.go @@ -125,6 +125,13 @@ var _ = Describe("Event signature generator", func() { Expect(expected).To(Equal(actual)) }) + + It("gets the vat grab method signature", func() { + expected := "grab(bytes32,bytes32,bytes32,bytes32,int256,int256)" + actual := shared.GetSolidityMethodSignature(shared.VatABI, "grab") + + Expect(expected).To(Equal(actual)) + }) }) Describe("it handles events", func() { diff --git a/pkg/transformers/test_data/mocks/vat_grab/converter.go b/pkg/transformers/test_data/mocks/vat_grab/converter.go new file mode 100644 index 00000000..8c210278 --- /dev/null +++ b/pkg/transformers/test_data/mocks/vat_grab/converter.go @@ -0,0 +1,22 @@ +package vat_grab + +import ( + "github.com/ethereum/go-ethereum/core/types" + + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_grab" +) + +type MockVatGrabConverter struct { + err error + PassedLogs []types.Log +} + +func (converter *MockVatGrabConverter) ToModels(ethLogs []types.Log) ([]vat_grab.VatGrabModel, error) { + converter.PassedLogs = ethLogs + return []vat_grab.VatGrabModel{test_data.VatGrabModel}, converter.err +} + +func (converter *MockVatGrabConverter) SetConverterError(e error) { + converter.err = e +} diff --git a/pkg/transformers/test_data/mocks/vat_grab/repository.go b/pkg/transformers/test_data/mocks/vat_grab/repository.go new file mode 100644 index 00000000..f60bf2d4 --- /dev/null +++ b/pkg/transformers/test_data/mocks/vat_grab/repository.go @@ -0,0 +1,56 @@ +package vat_grab + +import ( + . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_grab" +) + +type MockVatGrabRepository struct { + createErr error + markHeaderCheckedErr error + markHeaderCheckedPassedHeaderID int64 + missingHeaders []core.Header + missingHeadersErr error + PassedStartingBlockNumber int64 + PassedEndingBlockNumber int64 + PassedHeaderID int64 + PassedModels []vat_grab.VatGrabModel +} + +func (repository *MockVatGrabRepository) Create(headerID int64, models []vat_grab.VatGrabModel) error { + repository.PassedHeaderID = headerID + repository.PassedModels = models + return repository.createErr +} + +func (repository *MockVatGrabRepository) MarkHeaderChecked(headerID int64) error { + repository.markHeaderCheckedPassedHeaderID = headerID + return repository.markHeaderCheckedErr +} + +func (repository *MockVatGrabRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + repository.PassedStartingBlockNumber = startingBlockNumber + repository.PassedEndingBlockNumber = endingBlockNumber + return repository.missingHeaders, repository.missingHeadersErr +} + +func (repository *MockVatGrabRepository) SetMissingHeadersErr(e error) { + repository.missingHeadersErr = e +} + +func (repository *MockVatGrabRepository) SetMissingHeaders(headers []core.Header) { + repository.missingHeaders = headers +} + +func (repository *MockVatGrabRepository) AssertMarkHeaderCheckedCalledWith(i int64) { + Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i)) +} + +func (repository *MockVatGrabRepository) SetMarkHeaderCheckedErr(e error) { + repository.markHeaderCheckedErr = e +} + +func (repository *MockVatGrabRepository) SetCreateError(e error) { + repository.createErr = e +} diff --git a/pkg/transformers/test_data/vat_grab.go b/pkg/transformers/test_data/vat_grab.go new file mode 100644 index 00000000..66cade05 --- /dev/null +++ b/pkg/transformers/test_data/vat_grab.go @@ -0,0 +1,39 @@ +package test_data + +import ( + "encoding/json" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_grab" +) + +var EthVatGrabLog = types.Log{ + Address: common.HexToAddress(shared.VatContractAddress), + Topics: []common.Hash{ + common.HexToHash("0x3690ae4c00000000000000000000000000000000000000000000000000000000"), + common.HexToHash("0x66616b6520696c6b000000000000000000000000000000000000000000000000"), + common.HexToHash("0x07fa9ef6609ca7921112231f8f195138ebba2977000000000000000000000000"), + common.HexToHash("0x7340e006f4135ba6970d43bf43d88dcad4e7a8ca000000000000000000000000"), + }, + Data: hexutil.MustDecode("0x000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000c43690ae4c66616b6520696c6b00000000000000000000000000000000000000000000000007fa9ef6609ca7921112231f8f195138ebba29770000000000000000000000007340e006f4135ba6970d43bf43d88dcad4e7a8ca0000000000000000000000007526eb4f95e2a1394797cb38a921fb1eba09291b00000000000000000000000000000000000000000000000000000000000000000000000000000000069f6bc7000000000000000000000000000000000000000000000000000000000d3ed78e"), + BlockNumber: 23, + TxHash: common.HexToHash("0x7cb84c750ce4985f7811abf641d52ffcb35306d943081475226484cf1470c6fa"), + TxIndex: 4, + BlockHash: common.HexToHash("0xf5a367d560e14c4658ef85e4877e08b5560a4773b69b39f6b8025910b666fade"), + Index: 0, + Removed: false, +} + +var rawVatGrabLog, _ = json.Marshal(EthVatGrabLog) +var VatGrabModel = vat_grab.VatGrabModel{ + Ilk: "fake ilk", + Urn: "0x07Fa9eF6609cA7921112231F8f195138ebbA2977", + V: "0x7340e006f4135BA6970D43bf43d88DCAD4e7a8CA", + W: "0x7526EB4f95e2a1394797Cb38a921Fb1EbA09291B", + Dink: "111111111", + Dart: "222222222", + TransactionIndex: EthVatGrabLog.TxIndex, + Raw: rawVatGrabLog, +} diff --git a/pkg/transformers/transformers.go b/pkg/transformers/transformers.go index 1b966f8c..76f26167 100644 --- a/pkg/transformers/transformers.go +++ b/pkg/transformers/transformers.go @@ -37,6 +37,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/tend" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_grab" "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_init" "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_toll" "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_tune" @@ -64,6 +65,7 @@ var ( PitFileStabilityFeeTransformerInitializer = stability_fee.PitFileStabilityFeeTransformerInitializer{Config: pitFileConfig}.NewPitFileStabilityFeeTransformer PriceFeedTransformerInitializer = price_feeds.PriceFeedTransformerInitializer{Config: price_feeds.PriceFeedConfig}.NewPriceFeedTransformer TendTransformerInitializer = tend.TendTransformerInitializer{Config: tend.TendConfig}.NewTendTransformer + VatGrabTransformerInitializer = vat_grab.VatGrabTransformerInitializer{Config: vat_grab.VatGrabConfig}.NewVatGrabTransformer VatInitTransformerInitializer = vat_init.VatInitTransformerInitializer{Config: vat_init.VatInitConfig}.NewVatInitTransformer VatTollTransformerInitializer = vat_toll.VatTollTransformerInitializer{Config: vat_toll.VatTollConfig}.NewVatTollTransformer VatTuneTransformerInitializer = vat_tune.VatTuneTransformerInitializer{Config: vat_tune.VatTuneConfig}.NewVatTuneTransformer @@ -89,6 +91,7 @@ func TransformerInitializers() []shared.TransformerInitializer { PitFileStabilityFeeTransformerInitializer, PriceFeedTransformerInitializer, TendTransformerInitializer, + VatGrabTransformerInitializer, VatInitTransformerInitializer, VatTollTransformerInitializer, VatTuneTransformerInitializer, diff --git a/pkg/transformers/vat_grab/config.go b/pkg/transformers/vat_grab/config.go new file mode 100644 index 00000000..6ff6bd4c --- /dev/null +++ b/pkg/transformers/vat_grab/config.go @@ -0,0 +1,11 @@ +package vat_grab + +import "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + +var VatGrabConfig = shared.TransformerConfig{ + ContractAddresses: []string{shared.VatContractAddress}, + ContractAbi: shared.VatABI, + Topics: []string{shared.VatGrabSignature}, + StartingBlockNumber: 0, + EndingBlockNumber: 10000000, +} diff --git a/pkg/transformers/vat_grab/converter.go b/pkg/transformers/vat_grab/converter.go new file mode 100644 index 00000000..8c9568df --- /dev/null +++ b/pkg/transformers/vat_grab/converter.go @@ -0,0 +1,63 @@ +package vat_grab + +import ( + "bytes" + "encoding/json" + "errors" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "math/big" +) + +type Converter interface { + ToModels(ethLogs []types.Log) ([]VatGrabModel, error) +} + +type VatGrabConverter struct{} + +func (VatGrabConverter) ToModels(ethLogs []types.Log) ([]VatGrabModel, error) { + var models []VatGrabModel + for _, ethLog := range ethLogs { + err := verifyLog(ethLog) + if err != nil { + return nil, err + } + ilk := string(bytes.Trim(ethLog.Topics[1].Bytes(), "\x00")) + urn := common.BytesToAddress(ethLog.Topics[2].Bytes()[:common.AddressLength]) + v := common.BytesToAddress(ethLog.Topics[3].Bytes()[:common.AddressLength]) + wBytes := shared.GetDataBytesAtIndex(-3, ethLog.Data) + w := common.BytesToAddress(wBytes[:common.AddressLength]) + dinkBytes := shared.GetDataBytesAtIndex(-2, ethLog.Data) + dink := big.NewInt(0).SetBytes(dinkBytes).String() + dartBytes := shared.GetDataBytesAtIndex(-1, ethLog.Data) + dart := big.NewInt(0).SetBytes(dartBytes).String() + + raw, err := json.Marshal(ethLog) + if err != nil { + return nil, err + } + model := VatGrabModel{ + Ilk: ilk, + Urn: urn.String(), + V: v.String(), + W: w.String(), + Dink: dink, + Dart: dart, + TransactionIndex: ethLog.TxIndex, + Raw: raw, + } + models = append(models, model) + } + return models, nil +} + +func verifyLog(log types.Log) error { + if len(log.Topics) < 4 { + return errors.New("log missing topics") + } + if len(log.Data) < shared.DataItemLength { + return errors.New("log missing data") + } + return nil +} diff --git a/pkg/transformers/vat_grab/converter_test.go b/pkg/transformers/vat_grab/converter_test.go new file mode 100644 index 00000000..728bd561 --- /dev/null +++ b/pkg/transformers/vat_grab/converter_test.go @@ -0,0 +1,47 @@ +package vat_grab_test + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_grab" +) + +var _ = Describe("Vat grab converter", func() { + var converter vat_grab.Converter + + BeforeEach(func() { + converter = vat_grab.VatGrabConverter{} + }) + + It("returns err if log is missing topics", func() { + badLog := types.Log{ + Data: []byte{1, 1, 1, 1, 1}, + } + + _, err := converter.ToModels([]types.Log{badLog}) + + Expect(err).To(HaveOccurred()) + }) + + It("returns err if log is missing data", func() { + badLog := types.Log{ + Topics: []common.Hash{{}, {}, {}, {}}, + } + + _, err := converter.ToModels([]types.Log{badLog}) + + Expect(err).To(HaveOccurred()) + }) + + It("converts a log to an model", func() { + models, err := converter.ToModels([]types.Log{test_data.EthVatGrabLog}) + + Expect(err).NotTo(HaveOccurred()) + Expect(len(models)).To(Equal(1)) + Expect(models[0]).To(Equal(test_data.VatGrabModel)) + }) +}) diff --git a/pkg/transformers/vat_grab/model.go b/pkg/transformers/vat_grab/model.go new file mode 100644 index 00000000..dc19883f --- /dev/null +++ b/pkg/transformers/vat_grab/model.go @@ -0,0 +1,12 @@ +package vat_grab + +type VatGrabModel struct { + Ilk string + Urn string + V string + W string + Dink string + Dart string + TransactionIndex uint `db:"tx_idx"` + Raw []byte `db:"raw_log"` +} diff --git a/pkg/transformers/vat_grab/repository.go b/pkg/transformers/vat_grab/repository.go new file mode 100644 index 00000000..86f61117 --- /dev/null +++ b/pkg/transformers/vat_grab/repository.go @@ -0,0 +1,74 @@ +package vat_grab + +import ( + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" +) + +type Repository interface { + Create(headerID int64, models []VatGrabModel) error + MarkHeaderChecked(headerID int64) error + MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) +} + +type VatGrabRepository struct { + db *postgres.DB +} + +func NewVatGrabRepository(db *postgres.DB) VatGrabRepository { + return VatGrabRepository{ + db: db, + } +} + +func (repository VatGrabRepository) Create(headerID int64, models []VatGrabModel) error { + tx, err := repository.db.Begin() + if err != nil { + return err + } + for _, model := range models { + _, err = tx.Exec( + `INSERT into maker.vat_grab (header_id, ilk, urn, v, w, dink, dart, tx_idx, raw_log) + VALUES($1, $2, $3, $4, $5, $6::NUMERIC, $7::NUMERIC, $8, $9)`, + headerID, model.Ilk, model.Urn, model.V, model.W, model.Dink, model.Dart, model.TransactionIndex, model.Raw, + ) + if err != nil { + tx.Rollback() + return err + } + } + _, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, vat_grab_checked) + VALUES ($1, $2) + ON CONFLICT (header_id) DO + UPDATE SET vat_grab_checked = $2`, headerID, true) + if err != nil { + tx.Rollback() + return err + } + return tx.Commit() +} + +func (repository VatGrabRepository) MarkHeaderChecked(headerID int64) error { + _, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, vat_grab_checked) + VALUES ($1, $2) + ON CONFLICT (header_id) DO + UPDATE SET vat_grab_checked = $2`, headerID, true) + return err +} + +func (repository VatGrabRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + var result []core.Header + err := repository.db.Select( + &result, + `SELECT headers.id, headers.block_number FROM headers + LEFT JOIN checked_headers on headers.id = header_id + WHERE (header_id ISNULL OR vat_grab_checked IS FALSE) + AND headers.block_number >= $1 + AND headers.block_number <= $2 + AND headers.eth_node_fingerprint = $3`, + startingBlockNumber, + endingBlockNumber, + repository.db.Node.ID, + ) + return result, err +} diff --git a/pkg/transformers/vat_grab/repository_test.go b/pkg/transformers/vat_grab/repository_test.go new file mode 100644 index 00000000..fbf15802 --- /dev/null +++ b/pkg/transformers/vat_grab/repository_test.go @@ -0,0 +1,205 @@ +package vat_grab_test + +import ( + "database/sql" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_grab" + "github.com/vulcanize/vulcanizedb/test_config" +) + +var _ = Describe("Vat grab repository", func() { + Describe("Create", func() { + var ( + db *postgres.DB + vatGrabRepository vat_grab.Repository + err error + headerID int64 + ) + + BeforeEach(func() { + db = test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{}) + Expect(err).NotTo(HaveOccurred()) + vatGrabRepository = vat_grab.NewVatGrabRepository(db) + }) + + It("adds a vat grab event", func() { + err = vatGrabRepository.Create(headerID, []vat_grab.VatGrabModel{test_data.VatGrabModel}) + + Expect(err).NotTo(HaveOccurred()) + var dbVatGrab vat_grab.VatGrabModel + err = db.Get(&dbVatGrab, `SELECT ilk, urn, v, w, dink, dart, tx_idx, raw_log FROM maker.vat_grab WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(dbVatGrab.Ilk).To(Equal(test_data.VatGrabModel.Ilk)) + Expect(dbVatGrab.Urn).To(Equal(test_data.VatGrabModel.Urn)) + Expect(dbVatGrab.V).To(Equal(test_data.VatGrabModel.V)) + Expect(dbVatGrab.W).To(Equal(test_data.VatGrabModel.W)) + Expect(dbVatGrab.Dink).To(Equal(test_data.VatGrabModel.Dink)) + Expect(dbVatGrab.Dart).To(Equal(test_data.VatGrabModel.Dart)) + Expect(dbVatGrab.TransactionIndex).To(Equal(test_data.VatGrabModel.TransactionIndex)) + Expect(dbVatGrab.Raw).To(MatchJSON(test_data.VatGrabModel.Raw)) + }) + + It("marks header as checked for logs", func() { + err = vatGrabRepository.Create(headerID, []vat_grab.VatGrabModel{test_data.VatGrabModel}) + + Expect(err).NotTo(HaveOccurred()) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT vat_grab_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + + It("does not duplicate pit file vat_grab events", func() { + err = vatGrabRepository.Create(headerID, []vat_grab.VatGrabModel{test_data.VatGrabModel}) + + Expect(err).NotTo(HaveOccurred()) + err = vatGrabRepository.Create(headerID, []vat_grab.VatGrabModel{test_data.VatGrabModel}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint")) + }) + + It("removes pit file vat_grab if corresponding header is deleted", func() { + err = vatGrabRepository.Create(headerID, []vat_grab.VatGrabModel{test_data.VatGrabModel}) + Expect(err).NotTo(HaveOccurred()) + + _, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID) + + Expect(err).NotTo(HaveOccurred()) + var dbVatGrab vat_grab.VatGrabModel + err = db.Get(&dbVatGrab, `SELECT ilk, urn, v, w, dink, dart, tx_idx, raw_log FROM maker.vat_grab WHERE header_id = $1`, headerID) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(sql.ErrNoRows)) + }) + }) + + Describe("MarkHeaderChecked", func() { + var ( + db *postgres.DB + vatGrabRepository vat_grab.Repository + err error + headerID int64 + ) + + BeforeEach(func() { + db = test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{}) + Expect(err).NotTo(HaveOccurred()) + vatGrabRepository = vat_grab.NewVatGrabRepository(db) + }) + + It("creates a row for a new headerID", func() { + err = vatGrabRepository.MarkHeaderChecked(headerID) + + Expect(err).NotTo(HaveOccurred()) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT vat_grab_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + + It("updates row when headerID already exists", func() { + _, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID) + err = vatGrabRepository.MarkHeaderChecked(headerID) + + Expect(err).NotTo(HaveOccurred()) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT vat_grab_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + }) + + Describe("MissingHeaders", func() { + It("returns headers that haven't been checked", func() { + db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + startingBlockNumber := int64(1) + vatGrabBlockNumber := int64(2) + endingBlockNumber := int64(3) + blockNumbers := []int64{startingBlockNumber, vatGrabBlockNumber, endingBlockNumber, endingBlockNumber + 1} + var headerIDs []int64 + for _, n := range blockNumbers { + headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + headerIDs = append(headerIDs, headerID) + Expect(err).NotTo(HaveOccurred()) + } + vatGrabRepository := vat_grab.NewVatGrabRepository(db) + err := vatGrabRepository.MarkHeaderChecked(headerIDs[1]) + Expect(err).NotTo(HaveOccurred()) + + headers, err := vatGrabRepository.MissingHeaders(startingBlockNumber, endingBlockNumber) + + Expect(err).NotTo(HaveOccurred()) + Expect(len(headers)).To(Equal(2)) + Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber))) + Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber))) + }) + + It("only treats headers as checked if vat grab logs have been checked", func() { + db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + startingBlockNumber := int64(1) + vatGrabdBlockNumber := int64(2) + endingBlockNumber := int64(3) + blockNumbers := []int64{startingBlockNumber, vatGrabdBlockNumber, endingBlockNumber, endingBlockNumber + 1} + var headerIDs []int64 + for _, n := range blockNumbers { + headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + headerIDs = append(headerIDs, headerID) + Expect(err).NotTo(HaveOccurred()) + } + vatGrabRepository := vat_grab.NewVatGrabRepository(db) + _, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1]) + Expect(err).NotTo(HaveOccurred()) + + headers, err := vatGrabRepository.MissingHeaders(startingBlockNumber, endingBlockNumber) + + Expect(err).NotTo(HaveOccurred()) + Expect(len(headers)).To(Equal(3)) + Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatGrabdBlockNumber))) + Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatGrabdBlockNumber))) + Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatGrabdBlockNumber))) + }) + + It("only returns headers associated with the current node", func() { + db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + blockNumbers := []int64{1, 2, 3} + headerRepository := repositories.NewHeaderRepository(db) + dbTwo := test_config.NewTestDB(core.Node{ID: "second"}) + headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo) + var headerIDs []int64 + for _, n := range blockNumbers { + headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + Expect(err).NotTo(HaveOccurred()) + headerIDs = append(headerIDs, headerID) + _, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + Expect(err).NotTo(HaveOccurred()) + } + vatGrabRepository := vat_grab.NewVatGrabRepository(db) + vatGrabRepositoryTwo := vat_grab.NewVatGrabRepository(dbTwo) + err := vatGrabRepository.MarkHeaderChecked(headerIDs[0]) + Expect(err).NotTo(HaveOccurred()) + + nodeOneMissingHeaders, err := vatGrabRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) + Expect(err).NotTo(HaveOccurred()) + Expect(len(nodeOneMissingHeaders)).To(Equal(len(blockNumbers) - 1)) + + nodeTwoMissingHeaders, err := vatGrabRepositoryTwo.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) + Expect(err).NotTo(HaveOccurred()) + Expect(len(nodeTwoMissingHeaders)).To(Equal(len(blockNumbers))) + }) + }) +}) diff --git a/pkg/transformers/vat_grab/tranformer.go b/pkg/transformers/vat_grab/tranformer.go new file mode 100644 index 00000000..47dc2820 --- /dev/null +++ b/pkg/transformers/vat_grab/tranformer.go @@ -0,0 +1,63 @@ +package vat_grab + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "log" +) + +type VatGrabTransformerInitializer struct { + Config shared.TransformerConfig +} + +func (initializer VatGrabTransformerInitializer) NewVatGrabTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer { + converter := VatGrabConverter{} + fetcher := shared.NewFetcher(blockChain) + repository := NewVatGrabRepository(db) + return VatGrabTransformer{ + Config: initializer.Config, + Converter: converter, + Fetcher: fetcher, + Repository: repository, + } +} + +type VatGrabTransformer struct { + Config shared.TransformerConfig + Converter Converter + Fetcher shared.LogFetcher + Repository Repository +} + +func (transformer VatGrabTransformer) Execute() error { + missingHeaders, err := transformer.Repository.MissingHeaders(transformer.Config.StartingBlockNumber, transformer.Config.EndingBlockNumber) + if err != nil { + return err + } + log.Printf("Fetching vat init event logs for %d headers \n", len(missingHeaders)) + for _, header := range missingHeaders { + topics := [][]common.Hash{{common.HexToHash(shared.VatGrabSignature)}} + matchingLogs, err := transformer.Fetcher.FetchLogs(VatGrabConfig.ContractAddresses, topics, header.BlockNumber) + if err != nil { + return err + } + if len(matchingLogs) < 1 { + err = transformer.Repository.MarkHeaderChecked(header.Id) + if err != nil { + return err + } + continue + } + models, err := transformer.Converter.ToModels(matchingLogs) + if err != nil { + return err + } + err = transformer.Repository.Create(header.Id, models) + if err != nil { + return err + } + } + return nil +} diff --git a/pkg/transformers/vat_grab/transformer_test.go b/pkg/transformers/vat_grab/transformer_test.go new file mode 100644 index 00000000..a756b326 --- /dev/null +++ b/pkg/transformers/vat_grab/transformer_test.go @@ -0,0 +1,196 @@ +package vat_grab_test + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/fakes" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks" + vat_grab_mocks "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks/vat_grab" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_grab" +) + +var _ = Describe("Vat grab transformer", func() { + It("gets missing headers for block numbers specified in config", func() { + repository := &vat_grab_mocks.MockVatGrabRepository{} + transformer := vat_grab.VatGrabTransformer{ + Config: vat_grab.VatGrabConfig, + Fetcher: &mocks.MockLogFetcher{}, + Converter: &vat_grab_mocks.MockVatGrabConverter{}, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + Expect(repository.PassedStartingBlockNumber).To(Equal(vat_grab.VatGrabConfig.StartingBlockNumber)) + Expect(repository.PassedEndingBlockNumber).To(Equal(vat_grab.VatGrabConfig.EndingBlockNumber)) + }) + + It("returns error if repository returns error for missing headers", func() { + repository := &vat_grab_mocks.MockVatGrabRepository{} + repository.SetMissingHeadersErr(fakes.FakeError) + transformer := vat_grab.VatGrabTransformer{ + Fetcher: &mocks.MockLogFetcher{}, + Converter: &vat_grab_mocks.MockVatGrabConverter{}, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + + It("fetches logs for missing headers", func() { + fetcher := &mocks.MockLogFetcher{} + repository := &vat_grab_mocks.MockVatGrabRepository{} + repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}, {BlockNumber: 2}}) + transformer := vat_grab.VatGrabTransformer{ + Fetcher: fetcher, + Converter: &vat_grab_mocks.MockVatGrabConverter{}, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + Expect(fetcher.FetchedBlocks).To(Equal([]int64{1, 2})) + Expect(fetcher.FetchedContractAddresses).To(Equal([][]string{vat_grab.VatGrabConfig.ContractAddresses, vat_grab.VatGrabConfig.ContractAddresses})) + Expect(fetcher.FetchedTopics).To(Equal([][]common.Hash{{common.HexToHash(shared.VatGrabSignature)}})) + }) + + It("returns error if fetcher returns error", func() { + fetcher := &mocks.MockLogFetcher{} + fetcher.SetFetcherError(fakes.FakeError) + repository := &vat_grab_mocks.MockVatGrabRepository{} + repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) + transformer := vat_grab.VatGrabTransformer{ + Fetcher: fetcher, + Converter: &vat_grab_mocks.MockVatGrabConverter{}, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + + It("marks header checked if no logs returned", func() { + mockConverter := &vat_grab_mocks.MockVatGrabConverter{} + mockRepository := &vat_grab_mocks.MockVatGrabRepository{} + headerID := int64(123) + mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}}) + mockFetcher := &mocks.MockLogFetcher{} + transformer := vat_grab.VatGrabTransformer{ + Converter: mockConverter, + Fetcher: mockFetcher, + Repository: mockRepository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + mockRepository.AssertMarkHeaderCheckedCalledWith(headerID) + }) + + It("returns error if marking header checked returns err", func() { + mockConverter := &vat_grab_mocks.MockVatGrabConverter{} + mockRepository := &vat_grab_mocks.MockVatGrabRepository{} + mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}}) + mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError) + mockFetcher := &mocks.MockLogFetcher{} + transformer := vat_grab.VatGrabTransformer{ + Converter: mockConverter, + Fetcher: mockFetcher, + Repository: mockRepository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + + It("converts matching logs", func() { + converter := &vat_grab_mocks.MockVatGrabConverter{} + fetcher := &mocks.MockLogFetcher{} + fetcher.SetFetchedLogs([]types.Log{test_data.EthVatGrabLog}) + repository := &vat_grab_mocks.MockVatGrabRepository{} + repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) + transformer := vat_grab.VatGrabTransformer{ + Fetcher: fetcher, + Converter: converter, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + Expect(converter.PassedLogs).To(Equal([]types.Log{test_data.EthVatGrabLog})) + }) + + It("returns error if converter returns error", func() { + converter := &vat_grab_mocks.MockVatGrabConverter{} + converter.SetConverterError(fakes.FakeError) + fetcher := &mocks.MockLogFetcher{} + fetcher.SetFetchedLogs([]types.Log{test_data.EthVatGrabLog}) + repository := &vat_grab_mocks.MockVatGrabRepository{} + repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) + transformer := vat_grab.VatGrabTransformer{ + Fetcher: fetcher, + Converter: converter, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + + It("persists vat grab model", func() { + converter := &vat_grab_mocks.MockVatGrabConverter{} + fetcher := &mocks.MockLogFetcher{} + fetcher.SetFetchedLogs([]types.Log{test_data.EthVatGrabLog}) + repository := &vat_grab_mocks.MockVatGrabRepository{} + fakeHeader := core.Header{BlockNumber: 1, Id: 2} + repository.SetMissingHeaders([]core.Header{fakeHeader}) + transformer := vat_grab.VatGrabTransformer{ + Fetcher: fetcher, + Converter: converter, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id)) + Expect(repository.PassedModels).To(Equal([]vat_grab.VatGrabModel{test_data.VatGrabModel})) + }) + + It("returns error if repository returns error for create", func() { + converter := &vat_grab_mocks.MockVatGrabConverter{} + fetcher := &mocks.MockLogFetcher{} + fetcher.SetFetchedLogs([]types.Log{test_data.EthVatGrabLog}) + repository := &vat_grab_mocks.MockVatGrabRepository{} + repository.SetMissingHeaders([]core.Header{{BlockNumber: 1, Id: 2}}) + repository.SetCreateError(fakes.FakeError) + transformer := vat_grab.VatGrabTransformer{ + Fetcher: fetcher, + Converter: converter, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) +}) diff --git a/pkg/transformers/vat_grab/vat_grab_suite_test.go b/pkg/transformers/vat_grab/vat_grab_suite_test.go new file mode 100644 index 00000000..276331d9 --- /dev/null +++ b/pkg/transformers/vat_grab/vat_grab_suite_test.go @@ -0,0 +1,19 @@ +package vat_grab_test + +import ( + "io/ioutil" + "log" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestVatGrab(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "VatGrab Suite") +} + +var _ = BeforeSuite(func() { + log.SetOutput(ioutil.Discard) +}) diff --git a/test_config/test_config.go b/test_config/test_config.go index 6afd95ce..c6abd42e 100644 --- a/test_config/test_config.go +++ b/test_config/test_config.go @@ -89,6 +89,7 @@ func CleanTestDB(db *postgres.DB) { db.MustExec("DELETE FROM maker.pit_file_stability_fee") db.MustExec("DELETE FROM maker.price_feeds") db.MustExec("DELETE FROM maker.tend") + db.MustExec("DELETE FROM maker.vat_grab") db.MustExec("DELETE FROM maker.vat_init") db.MustExec("DELETE FROM maker.vat_toll") db.MustExec("DELETE FROM maker.vat_tune")