diff --git a/cmd/continuousLogSync.go b/cmd/continuousLogSync.go index 60c5f6b3..f89fdb9a 100644 --- a/cmd/continuousLogSync.go +++ b/cmd/continuousLogSync.go @@ -87,20 +87,28 @@ func buildTransformerInitializerMap() map[string]shared2.TransformerInitializer transformerInitializerMap := make(map[string]shared2.TransformerInitializer) transformerInitializerMap["bite"] = transformers.BiteTransformerInitializer + transformerInitializerMap["catFileChopLump"] = transformers.CatFileChopLumpTransformerInitializer + transformerInitializerMap["catFileFlip"] = transformers.CatFileFlipTransformerInitializer + transformerInitializerMap["catFilePitVow"] = transformers.CatFilePitVowTransformerInitializer transformerInitializerMap["deal"] = transformers.DealTransformerInitializer transformerInitializerMap["dent"] = transformers.DentTransformerInitializer transformerInitializerMap["dripDrip"] = transformers.DripDripTransformerInitializer transformerInitializerMap["dripFileIlk"] = transformers.DripFileIlkTransformerInitializer transformerInitializerMap["dripFileRepo"] = transformers.DripFileRepoTransformerInitializer + transformerInitializerMap["dripFileVow"] = transformers.DripFileVowTransfromerInitializer transformerInitializerMap["flipKick"] = transformers.FlipKickTransformerInitializer + transformerInitializerMap["flopKick"] = transformers.FlopKickTransformerInitializer transformerInitializerMap["frob"] = transformers.FrobTransformerInitializer transformerInitializerMap["pitFileDebtCeiling"] = transformers.PitFileDebtCeilingTransformerInitializer transformerInitializerMap["pitFileIlk"] = transformers.PitFileIlkTransformerInitializer transformerInitializerMap["pitFileStabilityFee"] = transformers.PitFileStabilityFeeTransformerInitializer transformerInitializerMap["priceFeed"] = transformers.PriceFeedTransformerInitializer transformerInitializerMap["tend"] = transformers.TendTransformerInitializer + transformerInitializerMap["vatGrab"] = transformers.VatGrabTransformerInitializer transformerInitializerMap["vatInit"] = transformers.VatInitTransformerInitializer transformerInitializerMap["vatFold"] = transformers.VatFoldTransformerInitializer + transformerInitializerMap["vatToll"] = transformers.VatTollTransformerInitializer + transformerInitializerMap["vatTune"] = transformers.VatTuneTransformerInitializer return transformerInitializerMap } diff --git a/db/migrations/1538580167_remove_bid_id_unique_constraint.down.sql b/db/migrations/1538580167_remove_bid_id_unique_constraint.down.sql new file mode 100644 index 00000000..33dc29cb --- /dev/null +++ b/db/migrations/1538580167_remove_bid_id_unique_constraint.down.sql @@ -0,0 +1,5 @@ +ALTER TABLE maker.tend + add constraint tend_bid_id_key unique (bid_id); + +ALTER TABLE maker.dent + add constraint dent_bid_id_key unique (bid_id); diff --git a/db/migrations/1538580167_remove_bid_id_unique_constraint.up.sql b/db/migrations/1538580167_remove_bid_id_unique_constraint.up.sql new file mode 100644 index 00000000..65af04c3 --- /dev/null +++ b/db/migrations/1538580167_remove_bid_id_unique_constraint.up.sql @@ -0,0 +1,5 @@ +ALTER TABLE maker.tend + drop constraint tend_bid_id_key; + +ALTER TABLE maker.dent + drop constraint dent_bid_id_key; diff --git a/db/migrations/1538679495_create_vat_toll.down.sql b/db/migrations/1538679495_create_vat_toll.down.sql new file mode 100644 index 00000000..5c291a92 --- /dev/null +++ b/db/migrations/1538679495_create_vat_toll.down.sql @@ -0,0 +1,3 @@ +DROP TABLE maker.vat_toll; +ALTER TABLE public.checked_headers + DROP COLUMN vat_toll_checked; \ No newline at end of file diff --git a/db/migrations/1538679495_create_vat_toll.up.sql b/db/migrations/1538679495_create_vat_toll.up.sql new file mode 100644 index 00000000..a1d4c87c --- /dev/null +++ b/db/migrations/1538679495_create_vat_toll.up.sql @@ -0,0 +1,13 @@ +CREATE TABLE maker.vat_toll ( + id SERIAL PRIMARY KEY, + header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE, + ilk TEXT, + urn TEXT, + take NUMERIC, + tx_idx INTEGER NOT NULL, + raw_log JSONB, + UNIQUE (header_id, tx_idx) +); + +ALTER TABLE public.checked_headers + ADD COLUMN vat_toll_checked BOOLEAN NOT NULL DEFAULT FALSE; \ No newline at end of file diff --git a/db/migrations/1538689084_create_vat_tune.down.sql b/db/migrations/1538689084_create_vat_tune.down.sql new file mode 100644 index 00000000..1d1a34ec --- /dev/null +++ b/db/migrations/1538689084_create_vat_tune.down.sql @@ -0,0 +1,3 @@ +DROP TABLE maker.vat_tune; +ALTER TABLE public.checked_headers + DROP COLUMN vat_tune_checked; \ No newline at end of file diff --git a/db/migrations/1538689084_create_vat_tune.up.sql b/db/migrations/1538689084_create_vat_tune.up.sql new file mode 100644 index 00000000..d308a7bd --- /dev/null +++ b/db/migrations/1538689084_create_vat_tune.up.sql @@ -0,0 +1,16 @@ +CREATE TABLE maker.vat_tune ( + id SERIAL PRIMARY KEY, + header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE, + ilk TEXT, + urn TEXT, + v TEXT, + w TEXT, + dink NUMERIC, + dart NUMERIC, + tx_idx INTEGER NOT NULL, + raw_log JSONB, + UNIQUE (header_id, tx_idx) +); + +ALTER TABLE public.checked_headers + ADD COLUMN vat_tune_checked BOOLEAN NOT NULL DEFAULT FALSE; \ No newline at end of file diff --git a/db/migrations/1539033620_create_vat_grab.down.sql b/db/migrations/1539033620_create_vat_grab.down.sql new file mode 100644 index 00000000..eade21a1 --- /dev/null +++ b/db/migrations/1539033620_create_vat_grab.down.sql @@ -0,0 +1,3 @@ +DROP TABLE maker.vat_grab; +ALTER TABLE public.checked_headers + DROP COLUMN vat_grab_checked; \ No newline at end of file diff --git a/db/migrations/1539033620_create_vat_grab.up.sql b/db/migrations/1539033620_create_vat_grab.up.sql new file mode 100644 index 00000000..60b66d87 --- /dev/null +++ b/db/migrations/1539033620_create_vat_grab.up.sql @@ -0,0 +1,16 @@ +CREATE TABLE maker.vat_grab ( + id SERIAL PRIMARY KEY, + header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE, + ilk TEXT, + urn TEXT, + v TEXT, + w TEXT, + dink NUMERIC, + dart NUMERIC, + tx_idx INTEGER NOT NULL, + raw_log JSONB, + UNIQUE (header_id, tx_idx) +); + +ALTER TABLE public.checked_headers + ADD COLUMN vat_grab_checked BOOLEAN NOT NULL DEFAULT FALSE; \ No newline at end of file diff --git a/db/schema.sql b/db/schema.sql index df89c393..d5b08e38 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -732,6 +732,44 @@ CREATE SEQUENCE maker.vat_fold_id_seq ALTER SEQUENCE maker.vat_fold_id_seq OWNED BY maker.vat_fold.id; +-- +-- Name: vat_grab; Type: TABLE; Schema: maker; Owner: - +-- + +CREATE TABLE maker.vat_grab ( + id integer NOT NULL, + header_id integer NOT NULL, + ilk text, + urn text, + v text, + w text, + dink numeric, + dart numeric, + tx_idx integer NOT NULL, + raw_log jsonb +); + + +-- +-- Name: vat_grab_id_seq; Type: SEQUENCE; Schema: maker; Owner: - +-- + +CREATE SEQUENCE maker.vat_grab_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: vat_grab_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: - +-- + +ALTER SEQUENCE maker.vat_grab_id_seq OWNED BY maker.vat_grab.id; + + -- -- Name: vat_init; Type: TABLE; Schema: maker; Owner: - -- @@ -765,6 +803,79 @@ CREATE SEQUENCE maker.vat_init_id_seq ALTER SEQUENCE maker.vat_init_id_seq OWNED BY maker.vat_init.id; +-- +-- Name: vat_toll; Type: TABLE; Schema: maker; Owner: - +-- + +CREATE TABLE maker.vat_toll ( + id integer NOT NULL, + header_id integer NOT NULL, + ilk text, + urn text, + take numeric, + tx_idx integer NOT NULL, + raw_log jsonb +); + + +-- +-- Name: vat_toll_id_seq; Type: SEQUENCE; Schema: maker; Owner: - +-- + +CREATE SEQUENCE maker.vat_toll_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: vat_toll_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: - +-- + +ALTER SEQUENCE maker.vat_toll_id_seq OWNED BY maker.vat_toll.id; + + +-- +-- Name: vat_tune; Type: TABLE; Schema: maker; Owner: - +-- + +CREATE TABLE maker.vat_tune ( + id integer NOT NULL, + header_id integer NOT NULL, + ilk text, + urn text, + v text, + w text, + dink numeric, + dart numeric, + tx_idx integer NOT NULL, + raw_log jsonb +); + + +-- +-- Name: vat_tune_id_seq; Type: SEQUENCE; Schema: maker; Owner: - +-- + +CREATE SEQUENCE maker.vat_tune_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: vat_tune_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: - +-- + +ALTER SEQUENCE maker.vat_tune_id_seq OWNED BY maker.vat_tune.id; + + -- -- Name: logs; Type: TABLE; Schema: public; Owner: - -- @@ -866,7 +977,10 @@ CREATE TABLE public.checked_headers ( pit_file_ilk_checked boolean DEFAULT false NOT NULL, pit_file_stability_fee_checked boolean DEFAULT false NOT NULL, vat_init_checked boolean DEFAULT false NOT NULL, - vat_fold_checked boolean DEFAULT false NOT NULL + vat_fold_checked boolean DEFAULT false NOT NULL, + vat_toll_checked boolean DEFAULT false NOT NULL, + vat_tune_checked boolean DEFAULT false NOT NULL, + vat_grab_checked boolean DEFAULT false NOT NULL ); @@ -1320,6 +1434,13 @@ ALTER TABLE ONLY maker.tend ALTER COLUMN id SET DEFAULT nextval('maker.tend_id_s ALTER TABLE ONLY maker.vat_fold ALTER COLUMN id SET DEFAULT nextval('maker.vat_fold_id_seq'::regclass); +-- +-- Name: vat_grab id; Type: DEFAULT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.vat_grab ALTER COLUMN id SET DEFAULT nextval('maker.vat_grab_id_seq'::regclass); + + -- -- Name: vat_init id; Type: DEFAULT; Schema: maker; Owner: - -- @@ -1327,6 +1448,20 @@ ALTER TABLE ONLY maker.vat_fold ALTER COLUMN id SET DEFAULT nextval('maker.vat_f ALTER TABLE ONLY maker.vat_init ALTER COLUMN id SET DEFAULT nextval('maker.vat_init_id_seq'::regclass); +-- +-- Name: vat_toll id; Type: DEFAULT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.vat_toll ALTER COLUMN id SET DEFAULT nextval('maker.vat_toll_id_seq'::regclass); + + +-- +-- Name: vat_tune id; Type: DEFAULT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.vat_tune ALTER COLUMN id SET DEFAULT nextval('maker.vat_tune_id_seq'::regclass); + + -- -- Name: blocks id; Type: DEFAULT; Schema: public; Owner: - -- @@ -1477,14 +1612,6 @@ ALTER TABLE ONLY maker.deal ADD CONSTRAINT deal_pkey PRIMARY KEY (id); --- --- Name: dent dent_bid_id_key; Type: CONSTRAINT; Schema: maker; Owner: - --- - -ALTER TABLE ONLY maker.dent - ADD CONSTRAINT dent_bid_id_key UNIQUE (bid_id); - - -- -- Name: dent dent_header_id_tx_idx_key; Type: CONSTRAINT; Schema: maker; Owner: - -- @@ -1677,14 +1804,6 @@ ALTER TABLE ONLY maker.price_feeds ADD CONSTRAINT price_feeds_pkey PRIMARY KEY (id); --- --- Name: tend tend_bid_id_key; Type: CONSTRAINT; Schema: maker; Owner: - --- - -ALTER TABLE ONLY maker.tend - ADD CONSTRAINT tend_bid_id_key UNIQUE (bid_id); - - -- -- Name: tend tend_header_id_tx_idx_key; Type: CONSTRAINT; Schema: maker; Owner: - -- @@ -1717,6 +1836,22 @@ ALTER TABLE ONLY maker.vat_fold ADD CONSTRAINT vat_fold_pkey PRIMARY KEY (id); +-- +-- Name: vat_grab vat_grab_header_id_tx_idx_key; Type: CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.vat_grab + ADD CONSTRAINT vat_grab_header_id_tx_idx_key UNIQUE (header_id, tx_idx); + + +-- +-- Name: vat_grab vat_grab_pkey; Type: CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.vat_grab + ADD CONSTRAINT vat_grab_pkey PRIMARY KEY (id); + + -- -- Name: vat_init vat_init_header_id_tx_idx_key; Type: CONSTRAINT; Schema: maker; Owner: - -- @@ -1733,6 +1868,38 @@ ALTER TABLE ONLY maker.vat_init ADD CONSTRAINT vat_init_pkey PRIMARY KEY (id); +-- +-- Name: vat_toll vat_toll_header_id_tx_idx_key; Type: CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.vat_toll + ADD CONSTRAINT vat_toll_header_id_tx_idx_key UNIQUE (header_id, tx_idx); + + +-- +-- Name: vat_toll vat_toll_pkey; Type: CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.vat_toll + ADD CONSTRAINT vat_toll_pkey PRIMARY KEY (id); + + +-- +-- Name: vat_tune vat_tune_header_id_tx_idx_key; Type: CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.vat_tune + ADD CONSTRAINT vat_tune_header_id_tx_idx_key UNIQUE (header_id, tx_idx); + + +-- +-- Name: vat_tune vat_tune_pkey; Type: CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.vat_tune + ADD CONSTRAINT vat_tune_pkey PRIMARY KEY (id); + + -- -- Name: blocks blocks_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -2039,6 +2206,14 @@ ALTER TABLE ONLY maker.vat_fold ADD CONSTRAINT vat_fold_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE; +-- +-- Name: vat_grab vat_grab_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.vat_grab + ADD CONSTRAINT vat_grab_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE; + + -- -- Name: vat_init vat_init_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - -- @@ -2047,6 +2222,22 @@ ALTER TABLE ONLY maker.vat_init ADD CONSTRAINT vat_init_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE; +-- +-- Name: vat_toll vat_toll_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.vat_toll + ADD CONSTRAINT vat_toll_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE; + + +-- +-- Name: vat_tune vat_tune_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.vat_tune + ADD CONSTRAINT vat_tune_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE; + + -- -- Name: transactions blocks_fk; Type: FK CONSTRAINT; Schema: public; Owner: - -- diff --git a/pkg/transformers/shared/constants.go b/pkg/transformers/shared/constants.go index 74983e92..c7647eb1 100644 --- a/pkg/transformers/shared/constants.go +++ b/pkg/transformers/shared/constants.go @@ -55,8 +55,11 @@ var ( pitFileIlkMethod = "file(bytes32,bytes32,uint256)" pitFileStabilityFeeMethod = GetSolidityMethodSignature(PitABI, "file") tendMethod = GetSolidityMethodSignature(FlipperABI, "tend") + vatGrabMethod = GetSolidityMethodSignature(VatABI, "grab") vatInitMethod = GetSolidityMethodSignature(VatABI, "init") vatFoldMethod = GetSolidityMethodSignature(VatABI, "fold") + vatTollMethod = GetSolidityMethodSignature(VatABI, "toll") + vatTuneMethod = GetSolidityMethodSignature(VatABI, "tune") BiteSignature = GetEventSignature(biteMethod) DealSignature = GetLogNoteSignature(dealMethod) @@ -76,6 +79,9 @@ var ( PitFileIlkSignature = GetLogNoteSignature(pitFileIlkMethod) PitFileStabilityFeeSignature = GetLogNoteSignature(pitFileStabilityFeeMethod) TendFunctionSignature = GetLogNoteSignature(tendMethod) + VatGrabSignature = GetLogNoteSignature(vatGrabMethod) VatInitSignature = GetLogNoteSignature(vatInitMethod) VatFoldSignature = GetLogNoteSignature(vatFoldMethod) + VatTollSignature = GetLogNoteSignature(vatTollMethod) + VatTuneSignature = GetLogNoteSignature(vatTuneMethod) ) diff --git a/pkg/transformers/shared/event_signature_generator_test.go b/pkg/transformers/shared/event_signature_generator_test.go index e19cdc3e..0ccec73d 100644 --- a/pkg/transformers/shared/event_signature_generator_test.go +++ b/pkg/transformers/shared/event_signature_generator_test.go @@ -132,6 +132,13 @@ var _ = Describe("Event signature generator", func() { Expect(expected).To(Equal(actual)) }) + + It("gets the vat grab method signature", func() { + expected := "grab(bytes32,bytes32,bytes32,bytes32,int256,int256)" + actual := shared.GetSolidityMethodSignature(shared.VatABI, "grab") + + Expect(expected).To(Equal(actual)) + }) }) Describe("it handles events", func() { diff --git a/pkg/transformers/shared/utilities.go b/pkg/transformers/shared/utilities.go index 83afd982..91f2ccaf 100644 --- a/pkg/transformers/shared/utilities.go +++ b/pkg/transformers/shared/utilities.go @@ -32,3 +32,15 @@ func BigIntToString(value *big.Int) string { return result } } + +func GetDataBytesAtIndex(n int, logData []byte) []byte { + switch { + case n == -1: + return logData[len(logData)-DataItemLength:] + case n == -2: + return logData[len(logData)-(2*DataItemLength) : len(logData)-DataItemLength] + case n == -3: + return logData[len(logData)-(3*DataItemLength) : len(logData)-(2*DataItemLength)] + } + return []byte{} +} diff --git a/pkg/transformers/shared/utilities_test.go b/pkg/transformers/shared/utilities_test.go new file mode 100644 index 00000000..03e5ddc0 --- /dev/null +++ b/pkg/transformers/shared/utilities_test.go @@ -0,0 +1,49 @@ +package shared_test + +import ( + "github.com/ethereum/go-ethereum/common/hexutil" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/ethereum/go-ethereum/common" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "math/big" +) + +var _ = Describe("Shared utilities", func() { + Describe("getting data at index", func() { + It("gets bytes for the last index in log data", func() { + logData := hexutil.MustDecode("0x000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000c45dd6471a66616b6520696c6b0000000000000000000000000000000000000000000000007d7bee5fcfd8028cf7b00876c5b1421c800561a6000000000000000000000000a3e37186e017747dba34042e83e3f76ad3cce9b00000000000000000000000000f243e26db94b5426032e6dfa6007802dea2a61400000000000000000000000000000000000000000000000000000000000000000000000000000000075bcd15000000000000000000000000000000000000000000000000000000003ade68b1") + bigIntBytes := big.NewInt(987654321).Bytes() + // big.Int.Bytes() does not include zero padding, but bytes in data index are of fixed length and include zero padding + expected := []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} + expected = append(expected, bigIntBytes...) + + actual := shared.GetDataBytesAtIndex(-1, logData) + + Expect(expected[:]).To(Equal(actual)) + }) + + It("gets bytes for the second-to-last index in log data", func() { + logData := hexutil.MustDecode("0x000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000c45dd6471a66616b6520696c6b0000000000000000000000000000000000000000000000007d7bee5fcfd8028cf7b00876c5b1421c800561a6000000000000000000000000a3e37186e017747dba34042e83e3f76ad3cce9b00000000000000000000000000f243e26db94b5426032e6dfa6007802dea2a61400000000000000000000000000000000000000000000000000000000000000000000000000000000075bcd15000000000000000000000000000000000000000000000000000000003ade68b1") + bigIntBytes := big.NewInt(123456789).Bytes() + expected := []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} + expected = append(expected, bigIntBytes...) + + actual := shared.GetDataBytesAtIndex(-2, logData) + + Expect(expected[:]).To(Equal(actual)) + }) + + It("gets bytes for the third-to-last index in log data", func() { + logData := hexutil.MustDecode("0x000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000c45dd6471a66616b6520696c6b0000000000000000000000000000000000000000000000007d7bee5fcfd8028cf7b00876c5b1421c800561a6000000000000000000000000a3e37186e017747dba34042e83e3f76ad3cce9b00000000000000000000000000f243e26db94b5426032e6dfa6007802dea2a61400000000000000000000000000000000000000000000000000000000000000000000000000000000075bcd15000000000000000000000000000000000000000000000000000000003ade68b1") + addressBytes := common.HexToAddress("0x0F243E26db94B5426032E6DFA6007802Dea2a614").Bytes() + // common.address.Bytes() returns [20]byte{}, need [32]byte{} + expected := append(addressBytes, []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}...) + + actual := shared.GetDataBytesAtIndex(-3, logData) + + Expect(expected[:]).To(Equal(actual)) + }) + }) +}) diff --git a/pkg/transformers/test_data/mocks/vat_grab/converter.go b/pkg/transformers/test_data/mocks/vat_grab/converter.go new file mode 100644 index 00000000..8c210278 --- /dev/null +++ b/pkg/transformers/test_data/mocks/vat_grab/converter.go @@ -0,0 +1,22 @@ +package vat_grab + +import ( + "github.com/ethereum/go-ethereum/core/types" + + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_grab" +) + +type MockVatGrabConverter struct { + err error + PassedLogs []types.Log +} + +func (converter *MockVatGrabConverter) ToModels(ethLogs []types.Log) ([]vat_grab.VatGrabModel, error) { + converter.PassedLogs = ethLogs + return []vat_grab.VatGrabModel{test_data.VatGrabModel}, converter.err +} + +func (converter *MockVatGrabConverter) SetConverterError(e error) { + converter.err = e +} diff --git a/pkg/transformers/test_data/mocks/vat_grab/repository.go b/pkg/transformers/test_data/mocks/vat_grab/repository.go new file mode 100644 index 00000000..f60bf2d4 --- /dev/null +++ b/pkg/transformers/test_data/mocks/vat_grab/repository.go @@ -0,0 +1,56 @@ +package vat_grab + +import ( + . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_grab" +) + +type MockVatGrabRepository struct { + createErr error + markHeaderCheckedErr error + markHeaderCheckedPassedHeaderID int64 + missingHeaders []core.Header + missingHeadersErr error + PassedStartingBlockNumber int64 + PassedEndingBlockNumber int64 + PassedHeaderID int64 + PassedModels []vat_grab.VatGrabModel +} + +func (repository *MockVatGrabRepository) Create(headerID int64, models []vat_grab.VatGrabModel) error { + repository.PassedHeaderID = headerID + repository.PassedModels = models + return repository.createErr +} + +func (repository *MockVatGrabRepository) MarkHeaderChecked(headerID int64) error { + repository.markHeaderCheckedPassedHeaderID = headerID + return repository.markHeaderCheckedErr +} + +func (repository *MockVatGrabRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + repository.PassedStartingBlockNumber = startingBlockNumber + repository.PassedEndingBlockNumber = endingBlockNumber + return repository.missingHeaders, repository.missingHeadersErr +} + +func (repository *MockVatGrabRepository) SetMissingHeadersErr(e error) { + repository.missingHeadersErr = e +} + +func (repository *MockVatGrabRepository) SetMissingHeaders(headers []core.Header) { + repository.missingHeaders = headers +} + +func (repository *MockVatGrabRepository) AssertMarkHeaderCheckedCalledWith(i int64) { + Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i)) +} + +func (repository *MockVatGrabRepository) SetMarkHeaderCheckedErr(e error) { + repository.markHeaderCheckedErr = e +} + +func (repository *MockVatGrabRepository) SetCreateError(e error) { + repository.createErr = e +} diff --git a/pkg/transformers/test_data/mocks/vat_toll/converter.go b/pkg/transformers/test_data/mocks/vat_toll/converter.go new file mode 100644 index 00000000..5d1c2604 --- /dev/null +++ b/pkg/transformers/test_data/mocks/vat_toll/converter.go @@ -0,0 +1,21 @@ +package vat_toll + +import ( + "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_toll" +) + +type MockVatTollConverter struct { + err error + PassedLogs []types.Log +} + +func (converter *MockVatTollConverter) ToModels(ethLogs []types.Log) ([]vat_toll.VatTollModel, error) { + converter.PassedLogs = ethLogs + return []vat_toll.VatTollModel{test_data.VatTollModel}, converter.err +} + +func (converter *MockVatTollConverter) SetConverterError(e error) { + converter.err = e +} diff --git a/pkg/transformers/test_data/mocks/vat_toll/repository.go b/pkg/transformers/test_data/mocks/vat_toll/repository.go new file mode 100644 index 00000000..094f45a9 --- /dev/null +++ b/pkg/transformers/test_data/mocks/vat_toll/repository.go @@ -0,0 +1,57 @@ +package vat_toll + +import ( + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_toll" +) + +type MockVatTollRepository struct { + createErr error + markHeaderCheckedErr error + markHeaderCheckedPassedHeaderID int64 + missingHeadersErr error + missingHeaders []core.Header + PassedStartingBlockNumber int64 + PassedEndingBlockNumber int64 + PassedHeaderID int64 + PassedModels []vat_toll.VatTollModel +} + +func (repository *MockVatTollRepository) Create(headerID int64, models []vat_toll.VatTollModel) error { + repository.PassedHeaderID = headerID + repository.PassedModels = models + return repository.createErr +} + +func (repository *MockVatTollRepository) MarkHeaderChecked(headerID int64) error { + repository.markHeaderCheckedPassedHeaderID = headerID + return repository.markHeaderCheckedErr +} + +func (repository *MockVatTollRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + repository.PassedStartingBlockNumber = startingBlockNumber + repository.PassedEndingBlockNumber = endingBlockNumber + return repository.missingHeaders, repository.missingHeadersErr +} + +func (repository *MockVatTollRepository) SetCreateError(e error) { + repository.createErr = e +} + +func (repository *MockVatTollRepository) SetMarkHeaderCheckedErr(e error) { + repository.markHeaderCheckedErr = e +} + +func (repository *MockVatTollRepository) SetMissingHeadersErr(e error) { + repository.missingHeadersErr = e +} + +func (repository *MockVatTollRepository) SetMissingHeaders(headers []core.Header) { + repository.missingHeaders = headers +} + +func (repository *MockVatTollRepository) AssertMarkHeaderCheckedCalledWith(i int64) { + Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i)) +} diff --git a/pkg/transformers/test_data/mocks/vat_tune/converter.go b/pkg/transformers/test_data/mocks/vat_tune/converter.go new file mode 100644 index 00000000..5dbef7fd --- /dev/null +++ b/pkg/transformers/test_data/mocks/vat_tune/converter.go @@ -0,0 +1,21 @@ +package vat_tune + +import ( + "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_tune" +) + +type MockVatTuneConverter struct { + err error + PassedLogs []types.Log +} + +func (converter *MockVatTuneConverter) ToModels(ethLogs []types.Log) ([]vat_tune.VatTuneModel, error) { + converter.PassedLogs = ethLogs + return []vat_tune.VatTuneModel{test_data.VatTuneModel}, converter.err +} + +func (converter *MockVatTuneConverter) SetConverterError(e error) { + converter.err = e +} diff --git a/pkg/transformers/test_data/mocks/vat_tune/repository.go b/pkg/transformers/test_data/mocks/vat_tune/repository.go new file mode 100644 index 00000000..164d145d --- /dev/null +++ b/pkg/transformers/test_data/mocks/vat_tune/repository.go @@ -0,0 +1,57 @@ +package vat_tune + +import ( + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_tune" +) + +type MockVatTuneRepository struct { + createErr error + markHeaderCheckedErr error + markHeaderCheckedPassedHeaderID int64 + missingHeaders []core.Header + missingHeadersErr error + PassedStartingBlockNumber int64 + PassedEndingBlockNumber int64 + PassedHeaderID int64 + PassedModels []vat_tune.VatTuneModel +} + +func (repository *MockVatTuneRepository) Create(headerID int64, models []vat_tune.VatTuneModel) error { + repository.PassedHeaderID = headerID + repository.PassedModels = models + return repository.createErr +} + +func (repository *MockVatTuneRepository) MarkHeaderChecked(headerID int64) error { + repository.markHeaderCheckedPassedHeaderID = headerID + return repository.markHeaderCheckedErr +} + +func (repository *MockVatTuneRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + repository.PassedStartingBlockNumber = startingBlockNumber + repository.PassedEndingBlockNumber = endingBlockNumber + return repository.missingHeaders, repository.missingHeadersErr +} + +func (repository *MockVatTuneRepository) SetMissingHeadersErr(e error) { + repository.missingHeadersErr = e +} + +func (repository *MockVatTuneRepository) SetMissingHeaders(headers []core.Header) { + repository.missingHeaders = headers +} + +func (repository *MockVatTuneRepository) AssertMarkHeaderCheckedCalledWith(i int64) { + Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i)) +} + +func (repository *MockVatTuneRepository) SetMarkHeaderCheckedErr(e error) { + repository.markHeaderCheckedErr = e +} + +func (repository *MockVatTuneRepository) SetCreateError(e error) { + repository.createErr = e +} diff --git a/pkg/transformers/test_data/vat_grab.go b/pkg/transformers/test_data/vat_grab.go new file mode 100644 index 00000000..66cade05 --- /dev/null +++ b/pkg/transformers/test_data/vat_grab.go @@ -0,0 +1,39 @@ +package test_data + +import ( + "encoding/json" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_grab" +) + +var EthVatGrabLog = types.Log{ + Address: common.HexToAddress(shared.VatContractAddress), + Topics: []common.Hash{ + common.HexToHash("0x3690ae4c00000000000000000000000000000000000000000000000000000000"), + common.HexToHash("0x66616b6520696c6b000000000000000000000000000000000000000000000000"), + common.HexToHash("0x07fa9ef6609ca7921112231f8f195138ebba2977000000000000000000000000"), + common.HexToHash("0x7340e006f4135ba6970d43bf43d88dcad4e7a8ca000000000000000000000000"), + }, + Data: hexutil.MustDecode("0x000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000c43690ae4c66616b6520696c6b00000000000000000000000000000000000000000000000007fa9ef6609ca7921112231f8f195138ebba29770000000000000000000000007340e006f4135ba6970d43bf43d88dcad4e7a8ca0000000000000000000000007526eb4f95e2a1394797cb38a921fb1eba09291b00000000000000000000000000000000000000000000000000000000000000000000000000000000069f6bc7000000000000000000000000000000000000000000000000000000000d3ed78e"), + BlockNumber: 23, + TxHash: common.HexToHash("0x7cb84c750ce4985f7811abf641d52ffcb35306d943081475226484cf1470c6fa"), + TxIndex: 4, + BlockHash: common.HexToHash("0xf5a367d560e14c4658ef85e4877e08b5560a4773b69b39f6b8025910b666fade"), + Index: 0, + Removed: false, +} + +var rawVatGrabLog, _ = json.Marshal(EthVatGrabLog) +var VatGrabModel = vat_grab.VatGrabModel{ + Ilk: "fake ilk", + Urn: "0x07Fa9eF6609cA7921112231F8f195138ebbA2977", + V: "0x7340e006f4135BA6970D43bf43d88DCAD4e7a8CA", + W: "0x7526EB4f95e2a1394797Cb38a921Fb1EbA09291B", + Dink: "111111111", + Dart: "222222222", + TransactionIndex: EthVatGrabLog.TxIndex, + Raw: rawVatGrabLog, +} diff --git a/pkg/transformers/test_data/vat_toll.go b/pkg/transformers/test_data/vat_toll.go new file mode 100644 index 00000000..bc66d62b --- /dev/null +++ b/pkg/transformers/test_data/vat_toll.go @@ -0,0 +1,36 @@ +package test_data + +import ( + "encoding/json" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_toll" + "math/big" +) + +var EthVatTollLog = types.Log{ + Address: common.HexToAddress("0x239E6f0AB02713f1F8AA90ebeDeD9FC66Dc96CD6"), + Topics: []common.Hash{ + common.HexToHash("0x09b7a0b500000000000000000000000000000000000000000000000000000000"), + common.HexToHash("0x66616b6520696c6b000000000000000000000000000000000000000000000000"), + common.HexToHash("0xa3e37186e017747dba34042e83e3f76ad3cce9b0000000000000000000000000"), + common.HexToHash("0x00000000000000000000000000000000000000000000000000000000075bcd15"), + }, + Data: hexutil.MustDecode("0x0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000006409b7a0b566616b6520696c6b000000000000000000000000000000000000000000000000a3e37186e017747dba34042e83e3f76ad3cce9b000000000000000000000000000000000000000000000000000000000000000000000000000000000075bcd15"), + BlockNumber: 21, + TxHash: common.HexToHash("0x0d59cb158b033ffdfb9a021d1e80bfbbcd99594c62c501897ccee446bcd33828"), + TxIndex: 2, + BlockHash: common.HexToHash("0xdbfda0ecb4ac6267c50be21db97874dc1203c22033e19733a6f67ca00b51dfc5"), + Index: 4, + Removed: false, +} + +var rawVatTollLog, _ = json.Marshal(EthVatTollLog) +var VatTollModel = vat_toll.VatTollModel{ + Ilk: "fake ilk", + Urn: "0xA3E37186E017747DbA34042e83e3F76Ad3CcE9b0", + Take: big.NewInt(123456789).String(), + TransactionIndex: EthVatTollLog.TxIndex, + Raw: rawVatTollLog, +} diff --git a/pkg/transformers/test_data/vat_tune.go b/pkg/transformers/test_data/vat_tune.go new file mode 100644 index 00000000..0cc6a03e --- /dev/null +++ b/pkg/transformers/test_data/vat_tune.go @@ -0,0 +1,41 @@ +package test_data + +import ( + "encoding/json" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" + + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_tune" +) + +var EthVatTuneLog = types.Log{ + Address: common.HexToAddress("0x239E6f0AB02713f1F8AA90ebeDeD9FC66Dc96CD6"), + Topics: []common.Hash{ + common.HexToHash("0x5dd6471a00000000000000000000000000000000000000000000000000000000"), + common.HexToHash("0x66616b6520696c6b000000000000000000000000000000000000000000000000"), + common.HexToHash("0x7d7bee5fcfd8028cf7b00876c5b1421c800561a6000000000000000000000000"), + common.HexToHash("0xa3e37186e017747dba34042e83e3f76ad3cce9b0000000000000000000000000"), + }, + Data: hexutil.MustDecode("0x000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000c45dd6471a66616b6520696c6b0000000000000000000000000000000000000000000000007d7bee5fcfd8028cf7b00876c5b1421c800561a6000000000000000000000000a3e37186e017747dba34042e83e3f76ad3cce9b00000000000000000000000000f243e26db94b5426032e6dfa6007802dea2a61400000000000000000000000000000000000000000000000000000000000000000000000000000000075bcd15000000000000000000000000000000000000000000000000000000003ade68b1"), + BlockNumber: 22, + TxHash: common.HexToHash("0x2b19ef3965420d2d5eb8792eafbd5e365b2866200cfc4473835971f6965f831c"), + TxIndex: 3, + BlockHash: common.HexToHash("0xf3a4a9ecb79a721421b347424dc9fa072ca762f3ba340557b54a205f058f59a6"), + Index: 6, + Removed: false, +} + +var rawVatTuneLog, _ = json.Marshal(EthVatTuneLog) +var VatTuneModel = vat_tune.VatTuneModel{ + Ilk: "fake ilk", + Urn: "0x7d7bEe5fCfD8028cf7b00876C5b1421c800561A6", + V: "0xA3E37186E017747DbA34042e83e3F76Ad3CcE9b0", + W: "0x0F243E26db94B5426032E6DFA6007802Dea2a614", + Dink: big.NewInt(123456789).String(), + Dart: big.NewInt(987654321).String(), + TransactionIndex: EthVatTuneLog.TxIndex, + Raw: rawVatTuneLog, +} diff --git a/pkg/transformers/transformers.go b/pkg/transformers/transformers.go index 38327842..de007146 100644 --- a/pkg/transformers/transformers.go +++ b/pkg/transformers/transformers.go @@ -38,7 +38,10 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/tend" "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_fold" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_grab" "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_init" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_toll" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_tune" ) var ( @@ -63,8 +66,11 @@ var ( PitFileStabilityFeeTransformerInitializer = stability_fee.PitFileStabilityFeeTransformerInitializer{Config: pitFileConfig}.NewPitFileStabilityFeeTransformer PriceFeedTransformerInitializer = price_feeds.PriceFeedTransformerInitializer{Config: price_feeds.PriceFeedConfig}.NewPriceFeedTransformer TendTransformerInitializer = tend.TendTransformerInitializer{Config: tend.TendConfig}.NewTendTransformer + VatGrabTransformerInitializer = vat_grab.VatGrabTransformerInitializer{Config: vat_grab.VatGrabConfig}.NewVatGrabTransformer VatInitTransformerInitializer = vat_init.VatInitTransformerInitializer{Config: vat_init.VatInitConfig}.NewVatInitTransformer VatFoldTransformerInitializer = vat_fold.VatFoldTransformerInitializer{Config: vat_fold.VatFoldConfig}.NewVatFoldTransformer + VatTollTransformerInitializer = vat_toll.VatTollTransformerInitializer{Config: vat_toll.VatTollConfig}.NewVatTollTransformer + VatTuneTransformerInitializer = vat_tune.VatTuneTransformerInitializer{Config: vat_tune.VatTuneConfig}.NewVatTuneTransformer ) func TransformerInitializers() []shared.TransformerInitializer { @@ -87,7 +93,10 @@ func TransformerInitializers() []shared.TransformerInitializer { PitFileStabilityFeeTransformerInitializer, PriceFeedTransformerInitializer, TendTransformerInitializer, + VatGrabTransformerInitializer, VatInitTransformerInitializer, VatFoldTransformerInitializer, + VatTollTransformerInitializer, + VatTuneTransformerInitializer, } } diff --git a/pkg/transformers/vat_grab/config.go b/pkg/transformers/vat_grab/config.go new file mode 100644 index 00000000..6ff6bd4c --- /dev/null +++ b/pkg/transformers/vat_grab/config.go @@ -0,0 +1,11 @@ +package vat_grab + +import "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + +var VatGrabConfig = shared.TransformerConfig{ + ContractAddresses: []string{shared.VatContractAddress}, + ContractAbi: shared.VatABI, + Topics: []string{shared.VatGrabSignature}, + StartingBlockNumber: 0, + EndingBlockNumber: 10000000, +} diff --git a/pkg/transformers/vat_grab/converter.go b/pkg/transformers/vat_grab/converter.go new file mode 100644 index 00000000..8c9568df --- /dev/null +++ b/pkg/transformers/vat_grab/converter.go @@ -0,0 +1,63 @@ +package vat_grab + +import ( + "bytes" + "encoding/json" + "errors" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "math/big" +) + +type Converter interface { + ToModels(ethLogs []types.Log) ([]VatGrabModel, error) +} + +type VatGrabConverter struct{} + +func (VatGrabConverter) ToModels(ethLogs []types.Log) ([]VatGrabModel, error) { + var models []VatGrabModel + for _, ethLog := range ethLogs { + err := verifyLog(ethLog) + if err != nil { + return nil, err + } + ilk := string(bytes.Trim(ethLog.Topics[1].Bytes(), "\x00")) + urn := common.BytesToAddress(ethLog.Topics[2].Bytes()[:common.AddressLength]) + v := common.BytesToAddress(ethLog.Topics[3].Bytes()[:common.AddressLength]) + wBytes := shared.GetDataBytesAtIndex(-3, ethLog.Data) + w := common.BytesToAddress(wBytes[:common.AddressLength]) + dinkBytes := shared.GetDataBytesAtIndex(-2, ethLog.Data) + dink := big.NewInt(0).SetBytes(dinkBytes).String() + dartBytes := shared.GetDataBytesAtIndex(-1, ethLog.Data) + dart := big.NewInt(0).SetBytes(dartBytes).String() + + raw, err := json.Marshal(ethLog) + if err != nil { + return nil, err + } + model := VatGrabModel{ + Ilk: ilk, + Urn: urn.String(), + V: v.String(), + W: w.String(), + Dink: dink, + Dart: dart, + TransactionIndex: ethLog.TxIndex, + Raw: raw, + } + models = append(models, model) + } + return models, nil +} + +func verifyLog(log types.Log) error { + if len(log.Topics) < 4 { + return errors.New("log missing topics") + } + if len(log.Data) < shared.DataItemLength { + return errors.New("log missing data") + } + return nil +} diff --git a/pkg/transformers/vat_grab/converter_test.go b/pkg/transformers/vat_grab/converter_test.go new file mode 100644 index 00000000..728bd561 --- /dev/null +++ b/pkg/transformers/vat_grab/converter_test.go @@ -0,0 +1,47 @@ +package vat_grab_test + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_grab" +) + +var _ = Describe("Vat grab converter", func() { + var converter vat_grab.Converter + + BeforeEach(func() { + converter = vat_grab.VatGrabConverter{} + }) + + It("returns err if log is missing topics", func() { + badLog := types.Log{ + Data: []byte{1, 1, 1, 1, 1}, + } + + _, err := converter.ToModels([]types.Log{badLog}) + + Expect(err).To(HaveOccurred()) + }) + + It("returns err if log is missing data", func() { + badLog := types.Log{ + Topics: []common.Hash{{}, {}, {}, {}}, + } + + _, err := converter.ToModels([]types.Log{badLog}) + + Expect(err).To(HaveOccurred()) + }) + + It("converts a log to an model", func() { + models, err := converter.ToModels([]types.Log{test_data.EthVatGrabLog}) + + Expect(err).NotTo(HaveOccurred()) + Expect(len(models)).To(Equal(1)) + Expect(models[0]).To(Equal(test_data.VatGrabModel)) + }) +}) diff --git a/pkg/transformers/vat_grab/model.go b/pkg/transformers/vat_grab/model.go new file mode 100644 index 00000000..dc19883f --- /dev/null +++ b/pkg/transformers/vat_grab/model.go @@ -0,0 +1,12 @@ +package vat_grab + +type VatGrabModel struct { + Ilk string + Urn string + V string + W string + Dink string + Dart string + TransactionIndex uint `db:"tx_idx"` + Raw []byte `db:"raw_log"` +} diff --git a/pkg/transformers/vat_grab/repository.go b/pkg/transformers/vat_grab/repository.go new file mode 100644 index 00000000..86f61117 --- /dev/null +++ b/pkg/transformers/vat_grab/repository.go @@ -0,0 +1,74 @@ +package vat_grab + +import ( + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" +) + +type Repository interface { + Create(headerID int64, models []VatGrabModel) error + MarkHeaderChecked(headerID int64) error + MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) +} + +type VatGrabRepository struct { + db *postgres.DB +} + +func NewVatGrabRepository(db *postgres.DB) VatGrabRepository { + return VatGrabRepository{ + db: db, + } +} + +func (repository VatGrabRepository) Create(headerID int64, models []VatGrabModel) error { + tx, err := repository.db.Begin() + if err != nil { + return err + } + for _, model := range models { + _, err = tx.Exec( + `INSERT into maker.vat_grab (header_id, ilk, urn, v, w, dink, dart, tx_idx, raw_log) + VALUES($1, $2, $3, $4, $5, $6::NUMERIC, $7::NUMERIC, $8, $9)`, + headerID, model.Ilk, model.Urn, model.V, model.W, model.Dink, model.Dart, model.TransactionIndex, model.Raw, + ) + if err != nil { + tx.Rollback() + return err + } + } + _, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, vat_grab_checked) + VALUES ($1, $2) + ON CONFLICT (header_id) DO + UPDATE SET vat_grab_checked = $2`, headerID, true) + if err != nil { + tx.Rollback() + return err + } + return tx.Commit() +} + +func (repository VatGrabRepository) MarkHeaderChecked(headerID int64) error { + _, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, vat_grab_checked) + VALUES ($1, $2) + ON CONFLICT (header_id) DO + UPDATE SET vat_grab_checked = $2`, headerID, true) + return err +} + +func (repository VatGrabRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + var result []core.Header + err := repository.db.Select( + &result, + `SELECT headers.id, headers.block_number FROM headers + LEFT JOIN checked_headers on headers.id = header_id + WHERE (header_id ISNULL OR vat_grab_checked IS FALSE) + AND headers.block_number >= $1 + AND headers.block_number <= $2 + AND headers.eth_node_fingerprint = $3`, + startingBlockNumber, + endingBlockNumber, + repository.db.Node.ID, + ) + return result, err +} diff --git a/pkg/transformers/vat_grab/repository_test.go b/pkg/transformers/vat_grab/repository_test.go new file mode 100644 index 00000000..fbf15802 --- /dev/null +++ b/pkg/transformers/vat_grab/repository_test.go @@ -0,0 +1,205 @@ +package vat_grab_test + +import ( + "database/sql" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_grab" + "github.com/vulcanize/vulcanizedb/test_config" +) + +var _ = Describe("Vat grab repository", func() { + Describe("Create", func() { + var ( + db *postgres.DB + vatGrabRepository vat_grab.Repository + err error + headerID int64 + ) + + BeforeEach(func() { + db = test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{}) + Expect(err).NotTo(HaveOccurred()) + vatGrabRepository = vat_grab.NewVatGrabRepository(db) + }) + + It("adds a vat grab event", func() { + err = vatGrabRepository.Create(headerID, []vat_grab.VatGrabModel{test_data.VatGrabModel}) + + Expect(err).NotTo(HaveOccurred()) + var dbVatGrab vat_grab.VatGrabModel + err = db.Get(&dbVatGrab, `SELECT ilk, urn, v, w, dink, dart, tx_idx, raw_log FROM maker.vat_grab WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(dbVatGrab.Ilk).To(Equal(test_data.VatGrabModel.Ilk)) + Expect(dbVatGrab.Urn).To(Equal(test_data.VatGrabModel.Urn)) + Expect(dbVatGrab.V).To(Equal(test_data.VatGrabModel.V)) + Expect(dbVatGrab.W).To(Equal(test_data.VatGrabModel.W)) + Expect(dbVatGrab.Dink).To(Equal(test_data.VatGrabModel.Dink)) + Expect(dbVatGrab.Dart).To(Equal(test_data.VatGrabModel.Dart)) + Expect(dbVatGrab.TransactionIndex).To(Equal(test_data.VatGrabModel.TransactionIndex)) + Expect(dbVatGrab.Raw).To(MatchJSON(test_data.VatGrabModel.Raw)) + }) + + It("marks header as checked for logs", func() { + err = vatGrabRepository.Create(headerID, []vat_grab.VatGrabModel{test_data.VatGrabModel}) + + Expect(err).NotTo(HaveOccurred()) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT vat_grab_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + + It("does not duplicate pit file vat_grab events", func() { + err = vatGrabRepository.Create(headerID, []vat_grab.VatGrabModel{test_data.VatGrabModel}) + + Expect(err).NotTo(HaveOccurred()) + err = vatGrabRepository.Create(headerID, []vat_grab.VatGrabModel{test_data.VatGrabModel}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint")) + }) + + It("removes pit file vat_grab if corresponding header is deleted", func() { + err = vatGrabRepository.Create(headerID, []vat_grab.VatGrabModel{test_data.VatGrabModel}) + Expect(err).NotTo(HaveOccurred()) + + _, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID) + + Expect(err).NotTo(HaveOccurred()) + var dbVatGrab vat_grab.VatGrabModel + err = db.Get(&dbVatGrab, `SELECT ilk, urn, v, w, dink, dart, tx_idx, raw_log FROM maker.vat_grab WHERE header_id = $1`, headerID) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(sql.ErrNoRows)) + }) + }) + + Describe("MarkHeaderChecked", func() { + var ( + db *postgres.DB + vatGrabRepository vat_grab.Repository + err error + headerID int64 + ) + + BeforeEach(func() { + db = test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{}) + Expect(err).NotTo(HaveOccurred()) + vatGrabRepository = vat_grab.NewVatGrabRepository(db) + }) + + It("creates a row for a new headerID", func() { + err = vatGrabRepository.MarkHeaderChecked(headerID) + + Expect(err).NotTo(HaveOccurred()) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT vat_grab_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + + It("updates row when headerID already exists", func() { + _, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID) + err = vatGrabRepository.MarkHeaderChecked(headerID) + + Expect(err).NotTo(HaveOccurred()) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT vat_grab_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + }) + + Describe("MissingHeaders", func() { + It("returns headers that haven't been checked", func() { + db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + startingBlockNumber := int64(1) + vatGrabBlockNumber := int64(2) + endingBlockNumber := int64(3) + blockNumbers := []int64{startingBlockNumber, vatGrabBlockNumber, endingBlockNumber, endingBlockNumber + 1} + var headerIDs []int64 + for _, n := range blockNumbers { + headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + headerIDs = append(headerIDs, headerID) + Expect(err).NotTo(HaveOccurred()) + } + vatGrabRepository := vat_grab.NewVatGrabRepository(db) + err := vatGrabRepository.MarkHeaderChecked(headerIDs[1]) + Expect(err).NotTo(HaveOccurred()) + + headers, err := vatGrabRepository.MissingHeaders(startingBlockNumber, endingBlockNumber) + + Expect(err).NotTo(HaveOccurred()) + Expect(len(headers)).To(Equal(2)) + Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber))) + Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber))) + }) + + It("only treats headers as checked if vat grab logs have been checked", func() { + db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + startingBlockNumber := int64(1) + vatGrabdBlockNumber := int64(2) + endingBlockNumber := int64(3) + blockNumbers := []int64{startingBlockNumber, vatGrabdBlockNumber, endingBlockNumber, endingBlockNumber + 1} + var headerIDs []int64 + for _, n := range blockNumbers { + headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + headerIDs = append(headerIDs, headerID) + Expect(err).NotTo(HaveOccurred()) + } + vatGrabRepository := vat_grab.NewVatGrabRepository(db) + _, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1]) + Expect(err).NotTo(HaveOccurred()) + + headers, err := vatGrabRepository.MissingHeaders(startingBlockNumber, endingBlockNumber) + + Expect(err).NotTo(HaveOccurred()) + Expect(len(headers)).To(Equal(3)) + Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatGrabdBlockNumber))) + Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatGrabdBlockNumber))) + Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatGrabdBlockNumber))) + }) + + It("only returns headers associated with the current node", func() { + db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + blockNumbers := []int64{1, 2, 3} + headerRepository := repositories.NewHeaderRepository(db) + dbTwo := test_config.NewTestDB(core.Node{ID: "second"}) + headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo) + var headerIDs []int64 + for _, n := range blockNumbers { + headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + Expect(err).NotTo(HaveOccurred()) + headerIDs = append(headerIDs, headerID) + _, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + Expect(err).NotTo(HaveOccurred()) + } + vatGrabRepository := vat_grab.NewVatGrabRepository(db) + vatGrabRepositoryTwo := vat_grab.NewVatGrabRepository(dbTwo) + err := vatGrabRepository.MarkHeaderChecked(headerIDs[0]) + Expect(err).NotTo(HaveOccurred()) + + nodeOneMissingHeaders, err := vatGrabRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) + Expect(err).NotTo(HaveOccurred()) + Expect(len(nodeOneMissingHeaders)).To(Equal(len(blockNumbers) - 1)) + + nodeTwoMissingHeaders, err := vatGrabRepositoryTwo.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) + Expect(err).NotTo(HaveOccurred()) + Expect(len(nodeTwoMissingHeaders)).To(Equal(len(blockNumbers))) + }) + }) +}) diff --git a/pkg/transformers/vat_grab/tranformer.go b/pkg/transformers/vat_grab/tranformer.go new file mode 100644 index 00000000..47dc2820 --- /dev/null +++ b/pkg/transformers/vat_grab/tranformer.go @@ -0,0 +1,63 @@ +package vat_grab + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "log" +) + +type VatGrabTransformerInitializer struct { + Config shared.TransformerConfig +} + +func (initializer VatGrabTransformerInitializer) NewVatGrabTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer { + converter := VatGrabConverter{} + fetcher := shared.NewFetcher(blockChain) + repository := NewVatGrabRepository(db) + return VatGrabTransformer{ + Config: initializer.Config, + Converter: converter, + Fetcher: fetcher, + Repository: repository, + } +} + +type VatGrabTransformer struct { + Config shared.TransformerConfig + Converter Converter + Fetcher shared.LogFetcher + Repository Repository +} + +func (transformer VatGrabTransformer) Execute() error { + missingHeaders, err := transformer.Repository.MissingHeaders(transformer.Config.StartingBlockNumber, transformer.Config.EndingBlockNumber) + if err != nil { + return err + } + log.Printf("Fetching vat init event logs for %d headers \n", len(missingHeaders)) + for _, header := range missingHeaders { + topics := [][]common.Hash{{common.HexToHash(shared.VatGrabSignature)}} + matchingLogs, err := transformer.Fetcher.FetchLogs(VatGrabConfig.ContractAddresses, topics, header.BlockNumber) + if err != nil { + return err + } + if len(matchingLogs) < 1 { + err = transformer.Repository.MarkHeaderChecked(header.Id) + if err != nil { + return err + } + continue + } + models, err := transformer.Converter.ToModels(matchingLogs) + if err != nil { + return err + } + err = transformer.Repository.Create(header.Id, models) + if err != nil { + return err + } + } + return nil +} diff --git a/pkg/transformers/vat_grab/transformer_test.go b/pkg/transformers/vat_grab/transformer_test.go new file mode 100644 index 00000000..a756b326 --- /dev/null +++ b/pkg/transformers/vat_grab/transformer_test.go @@ -0,0 +1,196 @@ +package vat_grab_test + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/fakes" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks" + vat_grab_mocks "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks/vat_grab" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_grab" +) + +var _ = Describe("Vat grab transformer", func() { + It("gets missing headers for block numbers specified in config", func() { + repository := &vat_grab_mocks.MockVatGrabRepository{} + transformer := vat_grab.VatGrabTransformer{ + Config: vat_grab.VatGrabConfig, + Fetcher: &mocks.MockLogFetcher{}, + Converter: &vat_grab_mocks.MockVatGrabConverter{}, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + Expect(repository.PassedStartingBlockNumber).To(Equal(vat_grab.VatGrabConfig.StartingBlockNumber)) + Expect(repository.PassedEndingBlockNumber).To(Equal(vat_grab.VatGrabConfig.EndingBlockNumber)) + }) + + It("returns error if repository returns error for missing headers", func() { + repository := &vat_grab_mocks.MockVatGrabRepository{} + repository.SetMissingHeadersErr(fakes.FakeError) + transformer := vat_grab.VatGrabTransformer{ + Fetcher: &mocks.MockLogFetcher{}, + Converter: &vat_grab_mocks.MockVatGrabConverter{}, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + + It("fetches logs for missing headers", func() { + fetcher := &mocks.MockLogFetcher{} + repository := &vat_grab_mocks.MockVatGrabRepository{} + repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}, {BlockNumber: 2}}) + transformer := vat_grab.VatGrabTransformer{ + Fetcher: fetcher, + Converter: &vat_grab_mocks.MockVatGrabConverter{}, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + Expect(fetcher.FetchedBlocks).To(Equal([]int64{1, 2})) + Expect(fetcher.FetchedContractAddresses).To(Equal([][]string{vat_grab.VatGrabConfig.ContractAddresses, vat_grab.VatGrabConfig.ContractAddresses})) + Expect(fetcher.FetchedTopics).To(Equal([][]common.Hash{{common.HexToHash(shared.VatGrabSignature)}})) + }) + + It("returns error if fetcher returns error", func() { + fetcher := &mocks.MockLogFetcher{} + fetcher.SetFetcherError(fakes.FakeError) + repository := &vat_grab_mocks.MockVatGrabRepository{} + repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) + transformer := vat_grab.VatGrabTransformer{ + Fetcher: fetcher, + Converter: &vat_grab_mocks.MockVatGrabConverter{}, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + + It("marks header checked if no logs returned", func() { + mockConverter := &vat_grab_mocks.MockVatGrabConverter{} + mockRepository := &vat_grab_mocks.MockVatGrabRepository{} + headerID := int64(123) + mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}}) + mockFetcher := &mocks.MockLogFetcher{} + transformer := vat_grab.VatGrabTransformer{ + Converter: mockConverter, + Fetcher: mockFetcher, + Repository: mockRepository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + mockRepository.AssertMarkHeaderCheckedCalledWith(headerID) + }) + + It("returns error if marking header checked returns err", func() { + mockConverter := &vat_grab_mocks.MockVatGrabConverter{} + mockRepository := &vat_grab_mocks.MockVatGrabRepository{} + mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}}) + mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError) + mockFetcher := &mocks.MockLogFetcher{} + transformer := vat_grab.VatGrabTransformer{ + Converter: mockConverter, + Fetcher: mockFetcher, + Repository: mockRepository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + + It("converts matching logs", func() { + converter := &vat_grab_mocks.MockVatGrabConverter{} + fetcher := &mocks.MockLogFetcher{} + fetcher.SetFetchedLogs([]types.Log{test_data.EthVatGrabLog}) + repository := &vat_grab_mocks.MockVatGrabRepository{} + repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) + transformer := vat_grab.VatGrabTransformer{ + Fetcher: fetcher, + Converter: converter, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + Expect(converter.PassedLogs).To(Equal([]types.Log{test_data.EthVatGrabLog})) + }) + + It("returns error if converter returns error", func() { + converter := &vat_grab_mocks.MockVatGrabConverter{} + converter.SetConverterError(fakes.FakeError) + fetcher := &mocks.MockLogFetcher{} + fetcher.SetFetchedLogs([]types.Log{test_data.EthVatGrabLog}) + repository := &vat_grab_mocks.MockVatGrabRepository{} + repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) + transformer := vat_grab.VatGrabTransformer{ + Fetcher: fetcher, + Converter: converter, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + + It("persists vat grab model", func() { + converter := &vat_grab_mocks.MockVatGrabConverter{} + fetcher := &mocks.MockLogFetcher{} + fetcher.SetFetchedLogs([]types.Log{test_data.EthVatGrabLog}) + repository := &vat_grab_mocks.MockVatGrabRepository{} + fakeHeader := core.Header{BlockNumber: 1, Id: 2} + repository.SetMissingHeaders([]core.Header{fakeHeader}) + transformer := vat_grab.VatGrabTransformer{ + Fetcher: fetcher, + Converter: converter, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id)) + Expect(repository.PassedModels).To(Equal([]vat_grab.VatGrabModel{test_data.VatGrabModel})) + }) + + It("returns error if repository returns error for create", func() { + converter := &vat_grab_mocks.MockVatGrabConverter{} + fetcher := &mocks.MockLogFetcher{} + fetcher.SetFetchedLogs([]types.Log{test_data.EthVatGrabLog}) + repository := &vat_grab_mocks.MockVatGrabRepository{} + repository.SetMissingHeaders([]core.Header{{BlockNumber: 1, Id: 2}}) + repository.SetCreateError(fakes.FakeError) + transformer := vat_grab.VatGrabTransformer{ + Fetcher: fetcher, + Converter: converter, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) +}) diff --git a/pkg/transformers/vat_grab/vat_grab_suite_test.go b/pkg/transformers/vat_grab/vat_grab_suite_test.go new file mode 100644 index 00000000..276331d9 --- /dev/null +++ b/pkg/transformers/vat_grab/vat_grab_suite_test.go @@ -0,0 +1,19 @@ +package vat_grab_test + +import ( + "io/ioutil" + "log" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestVatGrab(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "VatGrab Suite") +} + +var _ = BeforeSuite(func() { + log.SetOutput(ioutil.Discard) +}) diff --git a/pkg/transformers/vat_toll/config.go b/pkg/transformers/vat_toll/config.go new file mode 100644 index 00000000..dcace67e --- /dev/null +++ b/pkg/transformers/vat_toll/config.go @@ -0,0 +1,11 @@ +package vat_toll + +import "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + +var VatTollConfig = shared.TransformerConfig{ + ContractAddresses: []string{shared.VatContractAddress}, + ContractAbi: shared.VatABI, + Topics: []string{shared.VatTollSignature}, + StartingBlockNumber: 0, + EndingBlockNumber: 10000000, +} diff --git a/pkg/transformers/vat_toll/converter.go b/pkg/transformers/vat_toll/converter.go new file mode 100644 index 00000000..6998b423 --- /dev/null +++ b/pkg/transformers/vat_toll/converter.go @@ -0,0 +1,51 @@ +package vat_toll + +import ( + "bytes" + "encoding/json" + "errors" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" +) + +type Converter interface { + ToModels(ethLogs []types.Log) ([]VatTollModel, error) +} + +type VatTollConverter struct{} + +func (VatTollConverter) ToModels(ethLogs []types.Log) ([]VatTollModel, error) { + var models []VatTollModel + for _, ethLog := range ethLogs { + err := verifyLog(ethLog) + if err != nil { + return nil, err + } + ilk := string(bytes.Trim(ethLog.Topics[1].Bytes(), "\x00")) + urn := common.BytesToAddress(ethLog.Topics[2].Bytes()[:common.AddressLength]) + take := ethLog.Topics[3].Big() + + raw, err := json.Marshal(ethLog) + if err != nil { + return nil, err + } + model := VatTollModel{ + Ilk: ilk, + Urn: urn.String(), + Take: take.String(), + TransactionIndex: ethLog.TxIndex, + Raw: raw, + } + models = append(models, model) + } + return models, nil +} + +func verifyLog(log types.Log) error { + numTopicInValidLog := 4 + if len(log.Topics) < numTopicInValidLog { + return errors.New("log missing topics") + } + return nil +} diff --git a/pkg/transformers/vat_toll/converter_test.go b/pkg/transformers/vat_toll/converter_test.go new file mode 100644 index 00000000..94b049f3 --- /dev/null +++ b/pkg/transformers/vat_toll/converter_test.go @@ -0,0 +1,33 @@ +package vat_toll_test + +import ( + "github.com/ethereum/go-ethereum/core/types" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_toll" +) + +var _ = Describe("Vat toll converter", func() { + It("returns err if log is missing topics", func() { + converter := vat_toll.VatTollConverter{} + badLog := types.Log{ + Data: []byte{1, 1, 1, 1, 1}, + } + + _, err := converter.ToModels([]types.Log{badLog}) + + Expect(err).To(HaveOccurred()) + }) + + It("converts a log to an model", func() { + converter := vat_toll.VatTollConverter{} + + models, err := converter.ToModels([]types.Log{test_data.EthVatTollLog}) + + Expect(err).NotTo(HaveOccurred()) + Expect(len(models)).To(Equal(1)) + Expect(models[0]).To(Equal(test_data.VatTollModel)) + }) +}) diff --git a/pkg/transformers/vat_toll/model.go b/pkg/transformers/vat_toll/model.go new file mode 100644 index 00000000..a0660ad1 --- /dev/null +++ b/pkg/transformers/vat_toll/model.go @@ -0,0 +1,9 @@ +package vat_toll + +type VatTollModel struct { + Ilk string + Urn string + Take string + TransactionIndex uint `db:"tx_idx"` + Raw []byte `db:"raw_log"` +} diff --git a/pkg/transformers/vat_toll/repository.go b/pkg/transformers/vat_toll/repository.go new file mode 100644 index 00000000..2057c5e6 --- /dev/null +++ b/pkg/transformers/vat_toll/repository.go @@ -0,0 +1,74 @@ +package vat_toll + +import ( + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" +) + +type Repository interface { + Create(headerID int64, models []VatTollModel) error + MarkHeaderChecked(headerID int64) error + MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) +} + +type VatTollRepository struct { + db *postgres.DB +} + +func NewVatTollRepository(db *postgres.DB) VatTollRepository { + return VatTollRepository{ + db: db, + } +} + +func (repository VatTollRepository) Create(headerID int64, models []VatTollModel) error { + tx, err := repository.db.Begin() + if err != nil { + return err + } + for _, model := range models { + _, err = tx.Exec( + `INSERT into maker.vat_toll (header_id, ilk, urn, take, tx_idx, raw_log) + VALUES($1, $2, $3, $4::NUMERIC, $5, $6)`, + headerID, model.Ilk, model.Urn, model.Take, model.TransactionIndex, model.Raw, + ) + if err != nil { + tx.Rollback() + return err + } + } + _, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, vat_toll_checked) + VALUES ($1, $2) + ON CONFLICT (header_id) DO + UPDATE SET vat_toll_checked = $2`, headerID, true) + if err != nil { + tx.Rollback() + return err + } + return tx.Commit() +} + +func (repository VatTollRepository) MarkHeaderChecked(headerID int64) error { + _, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, vat_toll_checked) + VALUES ($1, $2) + ON CONFLICT (header_id) DO + UPDATE SET vat_toll_checked = $2`, headerID, true) + return err +} + +func (repository VatTollRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + var result []core.Header + err := repository.db.Select( + &result, + `SELECT headers.id, headers.block_number FROM headers + LEFT JOIN checked_headers on headers.id = header_id + WHERE (header_id ISNULL OR vat_toll_checked IS FALSE) + AND headers.block_number >= $1 + AND headers.block_number <= $2 + AND headers.eth_node_fingerprint = $3`, + startingBlockNumber, + endingBlockNumber, + repository.db.Node.ID, + ) + return result, err +} diff --git a/pkg/transformers/vat_toll/repository_test.go b/pkg/transformers/vat_toll/repository_test.go new file mode 100644 index 00000000..241ff475 --- /dev/null +++ b/pkg/transformers/vat_toll/repository_test.go @@ -0,0 +1,206 @@ +package vat_toll_test + +import ( + "database/sql" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_toll" + "github.com/vulcanize/vulcanizedb/test_config" +) + +var _ = Describe("Vat toll repository", func() { + Describe("Create", func() { + var ( + db *postgres.DB + vatTollRepository vat_toll.Repository + err error + headerID int64 + ) + + BeforeEach(func() { + db = test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{}) + Expect(err).NotTo(HaveOccurred()) + vatTollRepository = vat_toll.NewVatTollRepository(db) + }) + + It("adds a vat toll event", func() { + err = vatTollRepository.Create(headerID, []vat_toll.VatTollModel{test_data.VatTollModel}) + + Expect(err).NotTo(HaveOccurred()) + var dbVatToll vat_toll.VatTollModel + err = db.Get(&dbVatToll, `SELECT ilk, urn, take, tx_idx, raw_log FROM maker.vat_toll WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(dbVatToll.Ilk).To(Equal(test_data.VatTollModel.Ilk)) + Expect(dbVatToll.Urn).To(Equal(test_data.VatTollModel.Urn)) + Expect(dbVatToll.Take).To(Equal(test_data.VatTollModel.Take)) + Expect(dbVatToll.TransactionIndex).To(Equal(test_data.VatTollModel.TransactionIndex)) + Expect(dbVatToll.Raw).To(MatchJSON(test_data.VatTollModel.Raw)) + }) + + It("marks header as checked for logs", func() { + err = vatTollRepository.Create(headerID, []vat_toll.VatTollModel{test_data.VatTollModel}) + + Expect(err).NotTo(HaveOccurred()) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT vat_toll_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + + It("does not duplicate vat toll events", func() { + err = vatTollRepository.Create(headerID, []vat_toll.VatTollModel{test_data.VatTollModel}) + Expect(err).NotTo(HaveOccurred()) + + err = vatTollRepository.Create(headerID, []vat_toll.VatTollModel{test_data.VatTollModel}) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint")) + }) + + It("removes vat toll if corresponding header is deleted", func() { + err = vatTollRepository.Create(headerID, []vat_toll.VatTollModel{test_data.VatTollModel}) + Expect(err).NotTo(HaveOccurred()) + + _, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID) + + Expect(err).NotTo(HaveOccurred()) + var dbVatToll vat_toll.VatTollModel + err = db.Get(&dbVatToll, `SELECT ilk, urn, take, tx_idx, raw_log FROM maker.vat_toll WHERE header_id = $1`, headerID) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(sql.ErrNoRows)) + }) + }) + + Describe("MarkHeaderChecked", func() { + var ( + db *postgres.DB + vatTollRepository vat_toll.Repository + err error + headerID int64 + ) + + BeforeEach(func() { + db = test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{}) + Expect(err).NotTo(HaveOccurred()) + vatTollRepository = vat_toll.NewVatTollRepository(db) + }) + + It("creates a row for a new headerID", func() { + err = vatTollRepository.MarkHeaderChecked(headerID) + + Expect(err).NotTo(HaveOccurred()) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT vat_toll_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + + It("updates row when headerID already exists", func() { + _, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID) + + err = vatTollRepository.MarkHeaderChecked(headerID) + + Expect(err).NotTo(HaveOccurred()) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT vat_toll_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + }) + + Describe("MissingHeaders", func() { + It("returns headers that haven't been checked", func() { + db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + startingBlockNumber := int64(1) + pitFileBlockNumber := int64(2) + endingBlockNumber := int64(3) + blockNumbers := []int64{startingBlockNumber, pitFileBlockNumber, endingBlockNumber, endingBlockNumber + 1} + var headerIDs []int64 + for _, n := range blockNumbers { + headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + headerIDs = append(headerIDs, headerID) + Expect(err).NotTo(HaveOccurred()) + } + pitFileRepository := vat_toll.NewVatTollRepository(db) + err := pitFileRepository.MarkHeaderChecked(headerIDs[1]) + Expect(err).NotTo(HaveOccurred()) + + headers, err := pitFileRepository.MissingHeaders(startingBlockNumber, endingBlockNumber) + + Expect(err).NotTo(HaveOccurred()) + Expect(len(headers)).To(Equal(2)) + Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber))) + Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber))) + }) + + It("only treats headers as checked if vat toll logs have been checked", func() { + db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + startingBlockNumber := int64(1) + vatTolldBlockNumber := int64(2) + endingBlockNumber := int64(3) + blockNumbers := []int64{startingBlockNumber, vatTolldBlockNumber, endingBlockNumber, endingBlockNumber + 1} + var headerIDs []int64 + for _, n := range blockNumbers { + headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + headerIDs = append(headerIDs, headerID) + Expect(err).NotTo(HaveOccurred()) + } + vatTollRepository := vat_toll.NewVatTollRepository(db) + _, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1]) + Expect(err).NotTo(HaveOccurred()) + + headers, err := vatTollRepository.MissingHeaders(startingBlockNumber, endingBlockNumber) + + Expect(err).NotTo(HaveOccurred()) + Expect(len(headers)).To(Equal(3)) + Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatTolldBlockNumber))) + Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatTolldBlockNumber))) + Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatTolldBlockNumber))) + }) + + It("only returns headers associated with the current node", func() { + db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + blockNumbers := []int64{1, 2, 3} + headerRepository := repositories.NewHeaderRepository(db) + dbTwo := test_config.NewTestDB(core.Node{ID: "second"}) + headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo) + var headerIDs []int64 + for _, n := range blockNumbers { + headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + Expect(err).NotTo(HaveOccurred()) + headerIDs = append(headerIDs, headerID) + _, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + Expect(err).NotTo(HaveOccurred()) + } + pitFileRepository := vat_toll.NewVatTollRepository(db) + pitFileRepositoryTwo := vat_toll.NewVatTollRepository(dbTwo) + err := pitFileRepository.MarkHeaderChecked(headerIDs[0]) + Expect(err).NotTo(HaveOccurred()) + + nodeOneMissingHeaders, err := pitFileRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) + Expect(err).NotTo(HaveOccurred()) + Expect(len(nodeOneMissingHeaders)).To(Equal(len(blockNumbers) - 1)) + + nodeTwoMissingHeaders, err := pitFileRepositoryTwo.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) + Expect(err).NotTo(HaveOccurred()) + Expect(len(nodeTwoMissingHeaders)).To(Equal(len(blockNumbers))) + }) + }) +}) diff --git a/pkg/transformers/vat_toll/transformer.go b/pkg/transformers/vat_toll/transformer.go new file mode 100644 index 00000000..454e143a --- /dev/null +++ b/pkg/transformers/vat_toll/transformer.go @@ -0,0 +1,62 @@ +package vat_toll + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "log" +) + +type VatTollTransformer struct { + Config shared.TransformerConfig + Converter Converter + Fetcher shared.LogFetcher + Repository Repository +} + +type VatTollTransformerInitializer struct { + Config shared.TransformerConfig +} + +func (initializer VatTollTransformerInitializer) NewVatTollTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer { + converter := VatTollConverter{} + fetcher := shared.NewFetcher(blockChain) + repository := NewVatTollRepository(db) + return VatTollTransformer{ + Config: initializer.Config, + Converter: converter, + Fetcher: fetcher, + Repository: repository, + } +} + +func (transformer VatTollTransformer) Execute() error { + missingHeaders, err := transformer.Repository.MissingHeaders(transformer.Config.StartingBlockNumber, transformer.Config.EndingBlockNumber) + if err != nil { + return err + } + log.Printf("Fetching vat toll event logs for %d headers \n", len(missingHeaders)) + for _, header := range missingHeaders { + topics := [][]common.Hash{{common.HexToHash(shared.VatTollSignature)}} + matchingLogs, err := transformer.Fetcher.FetchLogs(VatTollConfig.ContractAddresses, topics, header.BlockNumber) + if err != nil { + return err + } + if len(matchingLogs) < 1 { + err = transformer.Repository.MarkHeaderChecked(header.Id) + if err != nil { + return err + } + } + models, err := transformer.Converter.ToModels(matchingLogs) + if err != nil { + return err + } + err = transformer.Repository.Create(header.Id, models) + if err != nil { + return err + } + } + return nil +} diff --git a/pkg/transformers/vat_toll/transformer_test.go b/pkg/transformers/vat_toll/transformer_test.go new file mode 100644 index 00000000..12a37f9d --- /dev/null +++ b/pkg/transformers/vat_toll/transformer_test.go @@ -0,0 +1,196 @@ +package vat_toll_test + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/fakes" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks" + vat_toll_mocks "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks/vat_toll" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_toll" +) + +var _ = Describe("Vat toll transformer", func() { + It("gets missing headers for block numbers specified in config", func() { + repository := &vat_toll_mocks.MockVatTollRepository{} + transformer := vat_toll.VatTollTransformer{ + Config: vat_toll.VatTollConfig, + Fetcher: &mocks.MockLogFetcher{}, + Converter: &vat_toll_mocks.MockVatTollConverter{}, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + Expect(repository.PassedStartingBlockNumber).To(Equal(vat_toll.VatTollConfig.StartingBlockNumber)) + Expect(repository.PassedEndingBlockNumber).To(Equal(vat_toll.VatTollConfig.EndingBlockNumber)) + }) + + It("returns error if repository returns error for missing headers", func() { + repository := &vat_toll_mocks.MockVatTollRepository{} + repository.SetMissingHeadersErr(fakes.FakeError) + transformer := vat_toll.VatTollTransformer{ + Fetcher: &mocks.MockLogFetcher{}, + Converter: &vat_toll_mocks.MockVatTollConverter{}, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + + It("fetches logs for missing headers", func() { + fetcher := &mocks.MockLogFetcher{} + repository := &vat_toll_mocks.MockVatTollRepository{} + repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}, {BlockNumber: 2}}) + transformer := vat_toll.VatTollTransformer{ + Fetcher: fetcher, + Converter: &vat_toll_mocks.MockVatTollConverter{}, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + Expect(fetcher.FetchedBlocks).To(Equal([]int64{1, 2})) + Expect(fetcher.FetchedContractAddresses).To(Equal([][]string{vat_toll.VatTollConfig.ContractAddresses, vat_toll.VatTollConfig.ContractAddresses})) + Expect(fetcher.FetchedTopics).To(Equal([][]common.Hash{{common.HexToHash(shared.VatTollSignature)}})) + }) + + It("returns error if fetcher returns error", func() { + fetcher := &mocks.MockLogFetcher{} + fetcher.SetFetcherError(fakes.FakeError) + repository := &vat_toll_mocks.MockVatTollRepository{} + repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) + transformer := vat_toll.VatTollTransformer{ + Fetcher: fetcher, + Converter: &vat_toll_mocks.MockVatTollConverter{}, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + + It("marks header checked if no logs returned", func() { + mockConverter := &vat_toll_mocks.MockVatTollConverter{} + mockRepository := &vat_toll_mocks.MockVatTollRepository{} + headerID := int64(123) + mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}}) + mockFetcher := &mocks.MockLogFetcher{} + transformer := vat_toll.VatTollTransformer{ + Converter: mockConverter, + Fetcher: mockFetcher, + Repository: mockRepository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + mockRepository.AssertMarkHeaderCheckedCalledWith(headerID) + }) + + It("returns error if marking header checked returns err", func() { + mockConverter := &vat_toll_mocks.MockVatTollConverter{} + mockRepository := &vat_toll_mocks.MockVatTollRepository{} + mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}}) + mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError) + mockFetcher := &mocks.MockLogFetcher{} + transformer := vat_toll.VatTollTransformer{ + Converter: mockConverter, + Fetcher: mockFetcher, + Repository: mockRepository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + + It("converts matching logs", func() { + converter := &vat_toll_mocks.MockVatTollConverter{} + fetcher := &mocks.MockLogFetcher{} + fetcher.SetFetchedLogs([]types.Log{test_data.EthVatTollLog}) + repository := &vat_toll_mocks.MockVatTollRepository{} + repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) + transformer := vat_toll.VatTollTransformer{ + Fetcher: fetcher, + Converter: converter, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + Expect(converter.PassedLogs).To(Equal([]types.Log{test_data.EthVatTollLog})) + }) + + It("returns error if converter returns error", func() { + converter := &vat_toll_mocks.MockVatTollConverter{} + converter.SetConverterError(fakes.FakeError) + fetcher := &mocks.MockLogFetcher{} + fetcher.SetFetchedLogs([]types.Log{test_data.EthVatTollLog}) + repository := &vat_toll_mocks.MockVatTollRepository{} + repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) + transformer := vat_toll.VatTollTransformer{ + Fetcher: fetcher, + Converter: converter, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + + It("persists vat toll model", func() { + converter := &vat_toll_mocks.MockVatTollConverter{} + fetcher := &mocks.MockLogFetcher{} + fetcher.SetFetchedLogs([]types.Log{test_data.EthVatTollLog}) + repository := &vat_toll_mocks.MockVatTollRepository{} + fakeHeader := core.Header{BlockNumber: 1, Id: 2} + repository.SetMissingHeaders([]core.Header{fakeHeader}) + transformer := vat_toll.VatTollTransformer{ + Fetcher: fetcher, + Converter: converter, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id)) + Expect(repository.PassedModels).To(Equal([]vat_toll.VatTollModel{test_data.VatTollModel})) + }) + + It("returns error if repository returns error for create", func() { + converter := &vat_toll_mocks.MockVatTollConverter{} + fetcher := &mocks.MockLogFetcher{} + fetcher.SetFetchedLogs([]types.Log{test_data.EthVatTollLog}) + repository := &vat_toll_mocks.MockVatTollRepository{} + repository.SetMissingHeaders([]core.Header{{BlockNumber: 1, Id: 2}}) + repository.SetCreateError(fakes.FakeError) + transformer := vat_toll.VatTollTransformer{ + Fetcher: fetcher, + Converter: converter, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) +}) diff --git a/pkg/transformers/vat_toll/vat_toll_suite_test.go b/pkg/transformers/vat_toll/vat_toll_suite_test.go new file mode 100644 index 00000000..861e1a01 --- /dev/null +++ b/pkg/transformers/vat_toll/vat_toll_suite_test.go @@ -0,0 +1,19 @@ +package vat_toll_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "io/ioutil" + "log" +) + +func TestVatToll(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "VatToll Suite") +} + +var _ = BeforeSuite(func() { + log.SetOutput(ioutil.Discard) +}) diff --git a/pkg/transformers/vat_tune/config.go b/pkg/transformers/vat_tune/config.go new file mode 100644 index 00000000..589f63f4 --- /dev/null +++ b/pkg/transformers/vat_tune/config.go @@ -0,0 +1,11 @@ +package vat_tune + +import "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + +var VatTuneConfig = shared.TransformerConfig{ + ContractAddresses: []string{shared.VatContractAddress}, + ContractAbi: shared.VatABI, + Topics: []string{shared.VatTuneSignature}, + StartingBlockNumber: 0, + EndingBlockNumber: 10000000, +} diff --git a/pkg/transformers/vat_tune/converter.go b/pkg/transformers/vat_tune/converter.go new file mode 100644 index 00000000..b01f2892 --- /dev/null +++ b/pkg/transformers/vat_tune/converter.go @@ -0,0 +1,65 @@ +package vat_tune + +import ( + "bytes" + "encoding/json" + "errors" + "math/big" + + "github.com/ethereum/go-ethereum/core/types" + + "github.com/ethereum/go-ethereum/common" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" +) + +type Converter interface { + ToModels(ethLogs []types.Log) ([]VatTuneModel, error) +} + +type VatTuneConverter struct{} + +func (VatTuneConverter) ToModels(ethLogs []types.Log) ([]VatTuneModel, error) { + var models []VatTuneModel + for _, ethLog := range ethLogs { + err := verifyLog(ethLog) + if err != nil { + return nil, err + } + ilk := string(bytes.Trim(ethLog.Topics[1].Bytes(), "\x00")) + urn := common.BytesToAddress(ethLog.Topics[2].Bytes()[:common.AddressLength]) + v := common.BytesToAddress(ethLog.Topics[3].Bytes()[:common.AddressLength]) + wBytes := shared.GetDataBytesAtIndex(-3, ethLog.Data) + w := common.BytesToAddress(wBytes[:common.AddressLength]) + dinkBytes := shared.GetDataBytesAtIndex(-2, ethLog.Data) + dink := big.NewInt(0).SetBytes(dinkBytes).String() + dartBytes := shared.GetDataBytesAtIndex(-1, ethLog.Data) + dart := big.NewInt(0).SetBytes(dartBytes).String() + + raw, err := json.Marshal(ethLog) + if err != nil { + return nil, err + } + model := VatTuneModel{ + Ilk: ilk, + Urn: urn.String(), + V: v.String(), + W: w.String(), + Dink: dink, + Dart: dart, + TransactionIndex: ethLog.TxIndex, + Raw: raw, + } + models = append(models, model) + } + return models, nil +} + +func verifyLog(log types.Log) error { + if len(log.Topics) < 4 { + return errors.New("log missing topics") + } + if len(log.Data) < shared.DataItemLength { + return errors.New("log missing data") + } + return nil +} diff --git a/pkg/transformers/vat_tune/converter_test.go b/pkg/transformers/vat_tune/converter_test.go new file mode 100644 index 00000000..09347aaa --- /dev/null +++ b/pkg/transformers/vat_tune/converter_test.go @@ -0,0 +1,45 @@ +package vat_tune_test + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_tune" +) + +var _ = Describe("Vat tune converter", func() { + It("returns err if log is missing topics", func() { + converter := vat_tune.VatTuneConverter{} + badLog := types.Log{ + Data: []byte{1, 1, 1, 1, 1}, + } + + _, err := converter.ToModels([]types.Log{badLog}) + + Expect(err).To(HaveOccurred()) + }) + + It("returns err if log is missing data", func() { + converter := vat_tune.VatTuneConverter{} + badLog := types.Log{ + Topics: []common.Hash{{}, {}, {}, {}}, + } + + _, err := converter.ToModels([]types.Log{badLog}) + + Expect(err).To(HaveOccurred()) + }) + + It("converts a log to an model", func() { + converter := vat_tune.VatTuneConverter{} + + models, err := converter.ToModels([]types.Log{test_data.EthVatTuneLog}) + + Expect(err).NotTo(HaveOccurred()) + Expect(len(models)).To(Equal(1)) + Expect(models[0]).To(Equal(test_data.VatTuneModel)) + }) +}) diff --git a/pkg/transformers/vat_tune/model.go b/pkg/transformers/vat_tune/model.go new file mode 100644 index 00000000..a625d1e0 --- /dev/null +++ b/pkg/transformers/vat_tune/model.go @@ -0,0 +1,12 @@ +package vat_tune + +type VatTuneModel struct { + Ilk string + Urn string + V string + W string + Dink string + Dart string + TransactionIndex uint `db:"tx_idx"` + Raw []byte `db:"raw_log"` +} diff --git a/pkg/transformers/vat_tune/repository.go b/pkg/transformers/vat_tune/repository.go new file mode 100644 index 00000000..10db10d5 --- /dev/null +++ b/pkg/transformers/vat_tune/repository.go @@ -0,0 +1,74 @@ +package vat_tune + +import ( + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" +) + +type Repository interface { + Create(headerID int64, models []VatTuneModel) error + MarkHeaderChecked(headerID int64) error + MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) +} + +type VatTuneRepository struct { + db *postgres.DB +} + +func NewVatTuneRepository(db *postgres.DB) VatTuneRepository { + return VatTuneRepository{ + db: db, + } +} + +func (repository VatTuneRepository) Create(headerID int64, models []VatTuneModel) error { + tx, err := repository.db.Begin() + if err != nil { + return err + } + for _, model := range models { + _, err = tx.Exec( + `INSERT into maker.vat_tune (header_id, ilk, urn, v, w, dink, dart, tx_idx, raw_log) + VALUES($1, $2, $3, $4, $5, $6::NUMERIC, $7::NUMERIC, $8, $9)`, + headerID, model.Ilk, model.Urn, model.V, model.W, model.Dink, model.Dart, model.TransactionIndex, model.Raw, + ) + if err != nil { + tx.Rollback() + return err + } + } + _, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, vat_tune_checked) + VALUES ($1, $2) + ON CONFLICT (header_id) DO + UPDATE SET vat_tune_checked = $2`, headerID, true) + if err != nil { + tx.Rollback() + return err + } + return tx.Commit() +} + +func (repository VatTuneRepository) MarkHeaderChecked(headerID int64) error { + _, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, vat_tune_checked) + VALUES ($1, $2) + ON CONFLICT (header_id) DO + UPDATE SET vat_tune_checked = $2`, headerID, true) + return err +} + +func (repository VatTuneRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + var result []core.Header + err := repository.db.Select( + &result, + `SELECT headers.id, headers.block_number FROM headers + LEFT JOIN checked_headers on headers.id = header_id + WHERE (header_id ISNULL OR vat_tune_checked IS FALSE) + AND headers.block_number >= $1 + AND headers.block_number <= $2 + AND headers.eth_node_fingerprint = $3`, + startingBlockNumber, + endingBlockNumber, + repository.db.Node.ID, + ) + return result, err +} diff --git a/pkg/transformers/vat_tune/repository_test.go b/pkg/transformers/vat_tune/repository_test.go new file mode 100644 index 00000000..a0e54ac6 --- /dev/null +++ b/pkg/transformers/vat_tune/repository_test.go @@ -0,0 +1,209 @@ +package vat_tune_test + +import ( + "database/sql" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_tune" + "github.com/vulcanize/vulcanizedb/test_config" +) + +var _ = Describe("Vat tune repository", func() { + Describe("Create", func() { + var ( + db *postgres.DB + vatTuneRepository vat_tune.Repository + err error + headerID int64 + ) + + BeforeEach(func() { + db = test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{}) + Expect(err).NotTo(HaveOccurred()) + vatTuneRepository = vat_tune.NewVatTuneRepository(db) + }) + + It("adds a vat tune event", func() { + err = vatTuneRepository.Create(headerID, []vat_tune.VatTuneModel{test_data.VatTuneModel}) + + Expect(err).NotTo(HaveOccurred()) + var dbVatTune vat_tune.VatTuneModel + err = db.Get(&dbVatTune, `SELECT ilk, urn, v, w, dink, dart, tx_idx, raw_log FROM maker.vat_tune WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(dbVatTune.Ilk).To(Equal(test_data.VatTuneModel.Ilk)) + Expect(dbVatTune.Urn).To(Equal(test_data.VatTuneModel.Urn)) + Expect(dbVatTune.V).To(Equal(test_data.VatTuneModel.V)) + Expect(dbVatTune.W).To(Equal(test_data.VatTuneModel.W)) + Expect(dbVatTune.Dink).To(Equal(test_data.VatTuneModel.Dink)) + Expect(dbVatTune.Dart).To(Equal(test_data.VatTuneModel.Dart)) + Expect(dbVatTune.TransactionIndex).To(Equal(test_data.VatTuneModel.TransactionIndex)) + Expect(dbVatTune.Raw).To(MatchJSON(test_data.VatTuneModel.Raw)) + }) + + It("marks header as checked for logs", func() { + err = vatTuneRepository.Create(headerID, []vat_tune.VatTuneModel{test_data.VatTuneModel}) + + Expect(err).NotTo(HaveOccurred()) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT vat_tune_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + + It("does not duplicate pit file vat_tune events", func() { + err = vatTuneRepository.Create(headerID, []vat_tune.VatTuneModel{test_data.VatTuneModel}) + Expect(err).NotTo(HaveOccurred()) + + err = vatTuneRepository.Create(headerID, []vat_tune.VatTuneModel{test_data.VatTuneModel}) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint")) + }) + + It("removes pit file vat_tune if corresponding header is deleted", func() { + err = vatTuneRepository.Create(headerID, []vat_tune.VatTuneModel{test_data.VatTuneModel}) + Expect(err).NotTo(HaveOccurred()) + + _, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID) + + Expect(err).NotTo(HaveOccurred()) + var dbVatTune vat_tune.VatTuneModel + err = db.Get(&dbVatTune, `SELECT ilk, urn, v, w, dink, dart, tx_idx, raw_log FROM maker.vat_tune WHERE header_id = $1`, headerID) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(sql.ErrNoRows)) + }) + }) + + Describe("MarkHeaderChecked", func() { + var ( + db *postgres.DB + vatTuneRepository vat_tune.Repository + err error + headerID int64 + ) + + BeforeEach(func() { + db = test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{}) + Expect(err).NotTo(HaveOccurred()) + vatTuneRepository = vat_tune.NewVatTuneRepository(db) + }) + + It("creates a row for a new headerID", func() { + err = vatTuneRepository.MarkHeaderChecked(headerID) + + Expect(err).NotTo(HaveOccurred()) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT vat_tune_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + + It("updates row when headerID already exists", func() { + _, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID) + + err = vatTuneRepository.MarkHeaderChecked(headerID) + + Expect(err).NotTo(HaveOccurred()) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT vat_tune_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + }) + + Describe("MissingHeaders", func() { + It("returns headers that haven't been checked", func() { + db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + startingBlockNumber := int64(1) + vatTuneBlockNumber := int64(2) + endingBlockNumber := int64(3) + blockNumbers := []int64{startingBlockNumber, vatTuneBlockNumber, endingBlockNumber, endingBlockNumber + 1} + var headerIDs []int64 + for _, n := range blockNumbers { + headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + headerIDs = append(headerIDs, headerID) + Expect(err).NotTo(HaveOccurred()) + } + vatTuneRepository := vat_tune.NewVatTuneRepository(db) + err := vatTuneRepository.MarkHeaderChecked(headerIDs[1]) + Expect(err).NotTo(HaveOccurred()) + + headers, err := vatTuneRepository.MissingHeaders(startingBlockNumber, endingBlockNumber) + + Expect(err).NotTo(HaveOccurred()) + Expect(len(headers)).To(Equal(2)) + Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber))) + Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber))) + }) + + It("only treats headers as checked if vat tune logs have been checked", func() { + db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + startingBlockNumber := int64(1) + vatTunedBlockNumber := int64(2) + endingBlockNumber := int64(3) + blockNumbers := []int64{startingBlockNumber, vatTunedBlockNumber, endingBlockNumber, endingBlockNumber + 1} + var headerIDs []int64 + for _, n := range blockNumbers { + headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + headerIDs = append(headerIDs, headerID) + Expect(err).NotTo(HaveOccurred()) + } + vatTuneRepository := vat_tune.NewVatTuneRepository(db) + _, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1]) + Expect(err).NotTo(HaveOccurred()) + + headers, err := vatTuneRepository.MissingHeaders(startingBlockNumber, endingBlockNumber) + + Expect(err).NotTo(HaveOccurred()) + Expect(len(headers)).To(Equal(3)) + Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatTunedBlockNumber))) + Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatTunedBlockNumber))) + Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatTunedBlockNumber))) + }) + + It("only returns headers associated with the current node", func() { + db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + blockNumbers := []int64{1, 2, 3} + headerRepository := repositories.NewHeaderRepository(db) + dbTwo := test_config.NewTestDB(core.Node{ID: "second"}) + headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo) + var headerIDs []int64 + for _, n := range blockNumbers { + headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + Expect(err).NotTo(HaveOccurred()) + headerIDs = append(headerIDs, headerID) + _, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + Expect(err).NotTo(HaveOccurred()) + } + vatTuneRepository := vat_tune.NewVatTuneRepository(db) + vatTuneRepositoryTwo := vat_tune.NewVatTuneRepository(dbTwo) + err := vatTuneRepository.MarkHeaderChecked(headerIDs[0]) + Expect(err).NotTo(HaveOccurred()) + + nodeOneMissingHeaders, err := vatTuneRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) + Expect(err).NotTo(HaveOccurred()) + Expect(len(nodeOneMissingHeaders)).To(Equal(len(blockNumbers) - 1)) + + nodeTwoMissingHeaders, err := vatTuneRepositoryTwo.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) + Expect(err).NotTo(HaveOccurred()) + Expect(len(nodeTwoMissingHeaders)).To(Equal(len(blockNumbers))) + }) + }) +}) diff --git a/pkg/transformers/vat_tune/transformer.go b/pkg/transformers/vat_tune/transformer.go new file mode 100644 index 00000000..571fbe07 --- /dev/null +++ b/pkg/transformers/vat_tune/transformer.go @@ -0,0 +1,62 @@ +package vat_tune + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "log" +) + +type VatTuneTransformerInitializer struct { + Config shared.TransformerConfig +} + +func (initializer VatTuneTransformerInitializer) NewVatTuneTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer { + converter := VatTuneConverter{} + fetcher := shared.NewFetcher(blockChain) + repository := NewVatTuneRepository(db) + return VatTuneTransformer{ + Config: initializer.Config, + Converter: converter, + Fetcher: fetcher, + Repository: repository, + } +} + +type VatTuneTransformer struct { + Config shared.TransformerConfig + Converter Converter + Fetcher shared.LogFetcher + Repository Repository +} + +func (transformer VatTuneTransformer) Execute() error { + missingHeaders, err := transformer.Repository.MissingHeaders(transformer.Config.StartingBlockNumber, transformer.Config.EndingBlockNumber) + if err != nil { + return err + } + log.Printf("Fetching vat init event logs for %d headers \n", len(missingHeaders)) + for _, header := range missingHeaders { + topics := [][]common.Hash{{common.HexToHash(shared.VatTuneSignature)}} + matchingLogs, err := transformer.Fetcher.FetchLogs(VatTuneConfig.ContractAddresses, topics, header.BlockNumber) + if err != nil { + return err + } + if len(matchingLogs) < 1 { + err = transformer.Repository.MarkHeaderChecked(header.Id) + if err != nil { + return err + } + } + models, err := transformer.Converter.ToModels(matchingLogs) + if err != nil { + return err + } + err = transformer.Repository.Create(header.Id, models) + if err != nil { + return err + } + } + return nil +} diff --git a/pkg/transformers/vat_tune/transformer_test.go b/pkg/transformers/vat_tune/transformer_test.go new file mode 100644 index 00000000..8fbc1036 --- /dev/null +++ b/pkg/transformers/vat_tune/transformer_test.go @@ -0,0 +1,196 @@ +package vat_tune_test + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/fakes" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks" + vat_tune_mocks "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks/vat_tune" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_tune" +) + +var _ = Describe("Vat tune transformer", func() { + It("gets missing headers for block numbers specified in config", func() { + repository := &vat_tune_mocks.MockVatTuneRepository{} + transformer := vat_tune.VatTuneTransformer{ + Config: vat_tune.VatTuneConfig, + Fetcher: &mocks.MockLogFetcher{}, + Converter: &vat_tune_mocks.MockVatTuneConverter{}, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + Expect(repository.PassedStartingBlockNumber).To(Equal(vat_tune.VatTuneConfig.StartingBlockNumber)) + Expect(repository.PassedEndingBlockNumber).To(Equal(vat_tune.VatTuneConfig.EndingBlockNumber)) + }) + + It("returns error if repository returns error for missing headers", func() { + repository := &vat_tune_mocks.MockVatTuneRepository{} + repository.SetMissingHeadersErr(fakes.FakeError) + transformer := vat_tune.VatTuneTransformer{ + Fetcher: &mocks.MockLogFetcher{}, + Converter: &vat_tune_mocks.MockVatTuneConverter{}, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + + It("fetches logs for missing headers", func() { + fetcher := &mocks.MockLogFetcher{} + repository := &vat_tune_mocks.MockVatTuneRepository{} + repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}, {BlockNumber: 2}}) + transformer := vat_tune.VatTuneTransformer{ + Fetcher: fetcher, + Converter: &vat_tune_mocks.MockVatTuneConverter{}, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + Expect(fetcher.FetchedBlocks).To(Equal([]int64{1, 2})) + Expect(fetcher.FetchedContractAddresses).To(Equal([][]string{vat_tune.VatTuneConfig.ContractAddresses, vat_tune.VatTuneConfig.ContractAddresses})) + Expect(fetcher.FetchedTopics).To(Equal([][]common.Hash{{common.HexToHash(shared.VatTuneSignature)}})) + }) + + It("returns error if fetcher returns error", func() { + fetcher := &mocks.MockLogFetcher{} + fetcher.SetFetcherError(fakes.FakeError) + repository := &vat_tune_mocks.MockVatTuneRepository{} + repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) + transformer := vat_tune.VatTuneTransformer{ + Fetcher: fetcher, + Converter: &vat_tune_mocks.MockVatTuneConverter{}, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + + It("marks header checked if no logs returned", func() { + mockConverter := &vat_tune_mocks.MockVatTuneConverter{} + mockRepository := &vat_tune_mocks.MockVatTuneRepository{} + headerID := int64(123) + mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}}) + mockFetcher := &mocks.MockLogFetcher{} + transformer := vat_tune.VatTuneTransformer{ + Converter: mockConverter, + Fetcher: mockFetcher, + Repository: mockRepository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + mockRepository.AssertMarkHeaderCheckedCalledWith(headerID) + }) + + It("returns error if marking header checked returns err", func() { + mockConverter := &vat_tune_mocks.MockVatTuneConverter{} + mockRepository := &vat_tune_mocks.MockVatTuneRepository{} + mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}}) + mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError) + mockFetcher := &mocks.MockLogFetcher{} + transformer := vat_tune.VatTuneTransformer{ + Converter: mockConverter, + Fetcher: mockFetcher, + Repository: mockRepository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + + It("converts matching logs", func() { + converter := &vat_tune_mocks.MockVatTuneConverter{} + fetcher := &mocks.MockLogFetcher{} + fetcher.SetFetchedLogs([]types.Log{test_data.EthVatTuneLog}) + repository := &vat_tune_mocks.MockVatTuneRepository{} + repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) + transformer := vat_tune.VatTuneTransformer{ + Fetcher: fetcher, + Converter: converter, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + Expect(converter.PassedLogs).To(Equal([]types.Log{test_data.EthVatTuneLog})) + }) + + It("returns error if converter returns error", func() { + converter := &vat_tune_mocks.MockVatTuneConverter{} + converter.SetConverterError(fakes.FakeError) + fetcher := &mocks.MockLogFetcher{} + fetcher.SetFetchedLogs([]types.Log{test_data.EthVatTuneLog}) + repository := &vat_tune_mocks.MockVatTuneRepository{} + repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) + transformer := vat_tune.VatTuneTransformer{ + Fetcher: fetcher, + Converter: converter, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + + It("persists vat tune model", func() { + converter := &vat_tune_mocks.MockVatTuneConverter{} + fetcher := &mocks.MockLogFetcher{} + fetcher.SetFetchedLogs([]types.Log{test_data.EthVatTuneLog}) + repository := &vat_tune_mocks.MockVatTuneRepository{} + fakeHeader := core.Header{BlockNumber: 1, Id: 2} + repository.SetMissingHeaders([]core.Header{fakeHeader}) + transformer := vat_tune.VatTuneTransformer{ + Fetcher: fetcher, + Converter: converter, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id)) + Expect(repository.PassedModels).To(Equal([]vat_tune.VatTuneModel{test_data.VatTuneModel})) + }) + + It("returns error if repository returns error for create", func() { + converter := &vat_tune_mocks.MockVatTuneConverter{} + fetcher := &mocks.MockLogFetcher{} + fetcher.SetFetchedLogs([]types.Log{test_data.EthVatTuneLog}) + repository := &vat_tune_mocks.MockVatTuneRepository{} + repository.SetMissingHeaders([]core.Header{{BlockNumber: 1, Id: 2}}) + repository.SetCreateError(fakes.FakeError) + transformer := vat_tune.VatTuneTransformer{ + Fetcher: fetcher, + Converter: converter, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) +}) diff --git a/pkg/transformers/vat_tune/vat_tune_suite_test.go b/pkg/transformers/vat_tune/vat_tune_suite_test.go new file mode 100644 index 00000000..c783052b --- /dev/null +++ b/pkg/transformers/vat_tune/vat_tune_suite_test.go @@ -0,0 +1,19 @@ +package vat_tune_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "io/ioutil" + "log" +) + +func TestVatTune(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "VatTune Suite") +} + +var _ = BeforeSuite(func() { + log.SetOutput(ioutil.Discard) +}) diff --git a/test_config/test_config.go b/test_config/test_config.go index a225a4bf..51e729d8 100644 --- a/test_config/test_config.go +++ b/test_config/test_config.go @@ -90,8 +90,11 @@ func CleanTestDB(db *postgres.DB) { db.MustExec("DELETE FROM maker.pit_file_stability_fee") db.MustExec("DELETE FROM maker.price_feeds") db.MustExec("DELETE FROM maker.tend") + db.MustExec("DELETE FROM maker.vat_grab") db.MustExec("DELETE FROM maker.vat_init") db.MustExec("DELETE FROM maker.vat_fold") + db.MustExec("DELETE FROM maker.vat_toll") + db.MustExec("DELETE FROM maker.vat_tune") db.MustExec("DELETE FROM receipts") db.MustExec("DELETE FROM transactions") db.MustExec("DELETE FROM watched_contracts")