diff --git a/cmd/backfillMakerLogs.go b/cmd/backfillMakerLogs.go index 5bbd65d0..73ce9d12 100644 --- a/cmd/backfillMakerLogs.go +++ b/cmd/backfillMakerLogs.go @@ -47,7 +47,7 @@ func backfillMakerLogs() { log.Fatal("Failed to initialize database.") } - watcher := shared.NewWatcher(db, blockChain) + watcher := shared.NewEventWatcher(db, blockChain) watcher.AddTransformers(transformers.TransformerInitializers()) err = watcher.Execute() diff --git a/cmd/continuousLogSync.go b/cmd/continuousLogSync.go index 59b36bc1..240915fd 100644 --- a/cmd/continuousLogSync.go +++ b/cmd/continuousLogSync.go @@ -62,7 +62,7 @@ func syncMakerLogs() { initializers := getTransformerInitializers(transformerNames) - watcher := shared.NewWatcher(db, blockChain) + watcher := shared.NewEventWatcher(db, blockChain) watcher.AddTransformers(initializers) for range ticker.C { diff --git a/cmd/parseStorageDiffs.go b/cmd/parseStorageDiffs.go new file mode 100644 index 00000000..753603f1 --- /dev/null +++ b/cmd/parseStorageDiffs.go @@ -0,0 +1,66 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "github.com/spf13/cobra" + "github.com/vulcanize/vulcanizedb/libraries/shared" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/fs" + "github.com/vulcanize/vulcanizedb/pkg/transformers" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/storage" + "log" +) + +// parseStorageDiffsCmd represents the parseStorageDiffs command +var parseStorageDiffsCmd = &cobra.Command{ + Use: "parseStorageDiffs", + Short: "Continuously ingest storage diffs from a CSV file", + Long: `Read storage diffs out of a CSV file that is constantly receiving +new rows from an Ethereum node. For example: + +./vulcanizedb parseStorageDiffs --config=environments/staging.toml + +Note that the path to your storage diffs must be configured in your toml +file under storageDiffsPath.`, + Run: func(cmd *cobra.Command, args []string) { + parseStorageDiffs() + }, +} + +func init() { + rootCmd.AddCommand(parseStorageDiffsCmd) +} + +func parseStorageDiffs() { + blockChain := getBlockChain() + db, err := postgres.NewDB(databaseConfig, blockChain.Node()) + if err != nil { + log.Fatal("Failed to initialize database: ", err) + } + + tailer := fs.FileTailer{Path: storageDiffsPath} + + // TODO: configure transformers + watcher := shared.NewStorageWatcher(tailer, db) + watcher.AddTransformers([]storage.TransformerInitializer{transformers.GetPitStorageTransformer().NewTransformer}) + + err = watcher.Execute() + if err != nil { + log.Fatal(err) + } +} diff --git a/cmd/root.go b/cmd/root.go index 458a5158..c0d7e7ef 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -40,6 +40,7 @@ var ( ipc string levelDbPath string startingBlockNumber int64 + storageDiffsPath string syncAll bool endingBlockNumber int64 ) @@ -64,6 +65,7 @@ func Execute() { func database(cmd *cobra.Command, args []string) { ipc = viper.GetString("client.ipcpath") levelDbPath = viper.GetString("client.leveldbpath") + storageDiffsPath = viper.GetString("filesystem.storageDiffsPath") databaseConfig = config.Database{ Name: viper.GetString("database.name"), Hostname: viper.GetString("database.hostname"), @@ -86,6 +88,7 @@ func init() { rootCmd.PersistentFlags().String("client-ipcPath", "", "location of geth.ipc file") rootCmd.PersistentFlags().String("client-levelDbPath", "", "location of levelDb chaindata") rootCmd.PersistentFlags().String("datadog-name", "vulcanize-test", "datadog service name") + rootCmd.PersistentFlags().String("filesystem-storageDiffsPath", "", "location of storage diffs csv file") viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name")) viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port")) @@ -95,6 +98,7 @@ func init() { viper.BindPFlag("client.ipcPath", rootCmd.PersistentFlags().Lookup("client-ipcPath")) viper.BindPFlag("client.levelDbPath", rootCmd.PersistentFlags().Lookup("client-levelDbPath")) viper.BindPFlag("datadog.name", rootCmd.PersistentFlags().Lookup("datadog-name")) + viper.BindPFlag("filesystem.storageDiffsPath", rootCmd.PersistentFlags().Lookup("filesystem-storageDiffsPath")) } func initConfig() { diff --git a/db/migrations/00071_create_pit_contract_storage_tables.sql b/db/migrations/00071_create_pit_contract_storage_tables.sql new file mode 100644 index 00000000..856b0da0 --- /dev/null +++ b/db/migrations/00071_create_pit_contract_storage_tables.sql @@ -0,0 +1,52 @@ +-- +goose Up +CREATE TABLE maker.pit_drip ( + id SERIAL PRIMARY KEY, + block_number BIGINT, + block_hash TEXT, + drip TEXT +); + +CREATE TABLE maker.pit_ilk_spot ( + id SERIAL PRIMARY KEY, + block_number BIGINT, + block_hash TEXT, + ilk TEXT, + spot NUMERIC NOT NULL +); + +CREATE TABLE maker.pit_ilk_line ( + id SERIAL PRIMARY KEY, + block_number BIGINT, + block_hash TEXT, + ilk TEXT, + line NUMERIC NOT NULL +); + +CREATE TABLE maker.pit_line ( + id SERIAL PRIMARY KEY, + block_number BIGINT, + block_hash TEXT, + line NUMERIC NOT NULL +); + +CREATE TABLE maker.pit_live ( + id SERIAL PRIMARY KEY, + block_number BIGINT, + block_hash TEXT, + live NUMERIC NOT NULL +); + +CREATE TABLE maker.pit_vat ( + id SERIAL PRIMARY KEY, + block_number BIGINT, + block_hash TEXT, + vat TEXT +); + +-- +goose Down +DROP TABLE maker.pit_drip; +DROP TABLE maker.pit_ilk_spot; +DROP TABLE maker.pit_ilk_line; +DROP TABLE maker.pit_line; +DROP TABLE maker.pit_live; +DROP TABLE maker.pit_vat; \ No newline at end of file diff --git a/db/schema.sql b/db/schema.sql index a7871549..2e0a6df6 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -3,7 +3,7 @@ -- -- Dumped from database version 10.5 --- Dumped by pg_dump version 11.1 +-- Dumped by pg_dump version 10.5 SET statement_timeout = 0; SET lock_timeout = 0; @@ -22,6 +22,20 @@ SET row_security = off; CREATE SCHEMA maker; +-- +-- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog; + + +-- +-- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language'; + + -- -- Name: notify_pricefeed(); Type: FUNCTION; Schema: public; Owner: - -- @@ -559,6 +573,38 @@ CREATE SEQUENCE maker.frob_id_seq ALTER SEQUENCE maker.frob_id_seq OWNED BY maker.frob.id; +-- +-- Name: pit_drip; Type: TABLE; Schema: maker; Owner: - +-- + +CREATE TABLE maker.pit_drip ( + id integer NOT NULL, + block_number bigint, + block_hash text, + drip text +); + + +-- +-- Name: pit_drip_id_seq; Type: SEQUENCE; Schema: maker; Owner: - +-- + +CREATE SEQUENCE maker.pit_drip_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: pit_drip_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: - +-- + +ALTER SEQUENCE maker.pit_drip_id_seq OWNED BY maker.pit_drip.id; + + -- -- Name: pit_file_debt_ceiling; Type: TABLE; Schema: maker; Owner: - -- @@ -630,6 +676,168 @@ CREATE SEQUENCE maker.pit_file_ilk_id_seq ALTER SEQUENCE maker.pit_file_ilk_id_seq OWNED BY maker.pit_file_ilk.id; +-- +-- Name: pit_ilk_line; Type: TABLE; Schema: maker; Owner: - +-- + +CREATE TABLE maker.pit_ilk_line ( + id integer NOT NULL, + block_number bigint, + block_hash text, + ilk text, + line numeric NOT NULL +); + + +-- +-- Name: pit_ilk_line_id_seq; Type: SEQUENCE; Schema: maker; Owner: - +-- + +CREATE SEQUENCE maker.pit_ilk_line_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: pit_ilk_line_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: - +-- + +ALTER SEQUENCE maker.pit_ilk_line_id_seq OWNED BY maker.pit_ilk_line.id; + + +-- +-- Name: pit_ilk_spot; Type: TABLE; Schema: maker; Owner: - +-- + +CREATE TABLE maker.pit_ilk_spot ( + id integer NOT NULL, + block_number bigint, + block_hash text, + ilk text, + spot numeric NOT NULL +); + + +-- +-- Name: pit_ilk_spot_id_seq; Type: SEQUENCE; Schema: maker; Owner: - +-- + +CREATE SEQUENCE maker.pit_ilk_spot_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: pit_ilk_spot_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: - +-- + +ALTER SEQUENCE maker.pit_ilk_spot_id_seq OWNED BY maker.pit_ilk_spot.id; + + +-- +-- Name: pit_line; Type: TABLE; Schema: maker; Owner: - +-- + +CREATE TABLE maker.pit_line ( + id integer NOT NULL, + block_number bigint, + block_hash text, + line numeric NOT NULL +); + + +-- +-- Name: pit_line_id_seq; Type: SEQUENCE; Schema: maker; Owner: - +-- + +CREATE SEQUENCE maker.pit_line_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: pit_line_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: - +-- + +ALTER SEQUENCE maker.pit_line_id_seq OWNED BY maker.pit_line.id; + + +-- +-- Name: pit_live; Type: TABLE; Schema: maker; Owner: - +-- + +CREATE TABLE maker.pit_live ( + id integer NOT NULL, + block_number bigint, + block_hash text, + live numeric NOT NULL +); + + +-- +-- Name: pit_live_id_seq; Type: SEQUENCE; Schema: maker; Owner: - +-- + +CREATE SEQUENCE maker.pit_live_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: pit_live_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: - +-- + +ALTER SEQUENCE maker.pit_live_id_seq OWNED BY maker.pit_live.id; + + +-- +-- Name: pit_vat; Type: TABLE; Schema: maker; Owner: - +-- + +CREATE TABLE maker.pit_vat ( + id integer NOT NULL, + block_number bigint, + block_hash text, + vat text +); + + +-- +-- Name: pit_vat_id_seq; Type: SEQUENCE; Schema: maker; Owner: - +-- + +CREATE SEQUENCE maker.pit_vat_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: pit_vat_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: - +-- + +ALTER SEQUENCE maker.pit_vat_id_seq OWNED BY maker.pit_vat.id; + + -- -- Name: price_feeds; Type: TABLE; Schema: maker; Owner: - -- @@ -1618,6 +1826,13 @@ ALTER TABLE ONLY maker.flop_kick ALTER COLUMN id SET DEFAULT nextval('maker.flop ALTER TABLE ONLY maker.frob ALTER COLUMN id SET DEFAULT nextval('maker.frob_id_seq'::regclass); +-- +-- Name: pit_drip id; Type: DEFAULT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.pit_drip ALTER COLUMN id SET DEFAULT nextval('maker.pit_drip_id_seq'::regclass); + + -- -- Name: pit_file_debt_ceiling id; Type: DEFAULT; Schema: maker; Owner: - -- @@ -1632,6 +1847,41 @@ ALTER TABLE ONLY maker.pit_file_debt_ceiling ALTER COLUMN id SET DEFAULT nextval ALTER TABLE ONLY maker.pit_file_ilk ALTER COLUMN id SET DEFAULT nextval('maker.pit_file_ilk_id_seq'::regclass); +-- +-- Name: pit_ilk_line id; Type: DEFAULT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.pit_ilk_line ALTER COLUMN id SET DEFAULT nextval('maker.pit_ilk_line_id_seq'::regclass); + + +-- +-- Name: pit_ilk_spot id; Type: DEFAULT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.pit_ilk_spot ALTER COLUMN id SET DEFAULT nextval('maker.pit_ilk_spot_id_seq'::regclass); + + +-- +-- Name: pit_line id; Type: DEFAULT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.pit_line ALTER COLUMN id SET DEFAULT nextval('maker.pit_line_id_seq'::regclass); + + +-- +-- Name: pit_live id; Type: DEFAULT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.pit_live ALTER COLUMN id SET DEFAULT nextval('maker.pit_live_id_seq'::regclass); + + +-- +-- Name: pit_vat id; Type: DEFAULT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.pit_vat ALTER COLUMN id SET DEFAULT nextval('maker.pit_vat_id_seq'::regclass); + + -- -- Name: price_feeds id; Type: DEFAULT; Schema: maker; Owner: - -- @@ -2017,6 +2267,14 @@ ALTER TABLE ONLY maker.frob ADD CONSTRAINT frob_pkey PRIMARY KEY (id); +-- +-- Name: pit_drip pit_drip_pkey; Type: CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.pit_drip + ADD CONSTRAINT pit_drip_pkey PRIMARY KEY (id); + + -- -- Name: pit_file_debt_ceiling pit_file_debt_ceiling_header_id_tx_idx_log_idx_key; Type: CONSTRAINT; Schema: maker; Owner: - -- @@ -2049,6 +2307,46 @@ ALTER TABLE ONLY maker.pit_file_ilk ADD CONSTRAINT pit_file_ilk_pkey PRIMARY KEY (id); +-- +-- Name: pit_ilk_line pit_ilk_line_pkey; Type: CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.pit_ilk_line + ADD CONSTRAINT pit_ilk_line_pkey PRIMARY KEY (id); + + +-- +-- Name: pit_ilk_spot pit_ilk_spot_pkey; Type: CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.pit_ilk_spot + ADD CONSTRAINT pit_ilk_spot_pkey PRIMARY KEY (id); + + +-- +-- Name: pit_line pit_line_pkey; Type: CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.pit_line + ADD CONSTRAINT pit_line_pkey PRIMARY KEY (id); + + +-- +-- Name: pit_live pit_live_pkey; Type: CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.pit_live + ADD CONSTRAINT pit_live_pkey PRIMARY KEY (id); + + +-- +-- Name: pit_vat pit_vat_pkey; Type: CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.pit_vat + ADD CONSTRAINT pit_vat_pkey PRIMARY KEY (id); + + -- -- Name: price_feeds price_feeds_header_id_medianizer_address_tx_idx_log_idx_key; Type: CONSTRAINT; Schema: maker; Owner: - -- @@ -2677,4 +2975,5 @@ ALTER TABLE ONLY public.logs -- -- PostgreSQL database dump complete --- \ No newline at end of file +-- + diff --git a/environments/staging.toml b/environments/staging.toml index 51569120..8954a483 100644 --- a/environments/staging.toml +++ b/environments/staging.toml @@ -11,6 +11,9 @@ [datadog] name = "maker_vdb_staging" +[filesystem] + storageDiffsPath = "INSERT-PATH-TO-STORAGE-DIFFS" + [contract] [contract.address] cat = "0x2f34f22a00ee4b7a8f8bbc4eaee1658774c624e0" diff --git a/libraries/shared/watcher.go b/libraries/shared/event_watcher.go similarity index 93% rename from libraries/shared/watcher.go rename to libraries/shared/event_watcher.go index eddf11c9..d17288be 100644 --- a/libraries/shared/watcher.go +++ b/libraries/shared/event_watcher.go @@ -25,7 +25,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" ) -type Watcher struct { +type EventWatcher struct { Transformers []shared.Transformer DB *postgres.DB Fetcher shared.LogFetcher @@ -35,10 +35,10 @@ type Watcher struct { StartingBlock *int64 } -func NewWatcher(db *postgres.DB, bc core.BlockChain) Watcher { +func NewEventWatcher(db *postgres.DB, bc core.BlockChain) EventWatcher { chunker := shared.NewLogChunker() fetcher := shared.NewFetcher(bc) - return Watcher{ + return EventWatcher{ DB: db, Fetcher: fetcher, Chunker: chunker, @@ -46,7 +46,7 @@ func NewWatcher(db *postgres.DB, bc core.BlockChain) Watcher { } // Adds transformers to the watcher and updates the chunker, so that it will consider the new transformers. -func (watcher *Watcher) AddTransformers(initializers []shared.TransformerInitializer) { +func (watcher *EventWatcher) AddTransformers(initializers []shared.TransformerInitializer) { var contractAddresses []common.Address var topic0s []common.Hash var configs []shared.TransformerConfig @@ -74,7 +74,7 @@ func (watcher *Watcher) AddTransformers(initializers []shared.TransformerInitial watcher.Chunker.AddConfigs(configs) } -func (watcher *Watcher) Execute() error { +func (watcher *EventWatcher) Execute() error { if watcher.Transformers == nil { return fmt.Errorf("No transformers added to watcher") } diff --git a/libraries/shared/watcher_test.go b/libraries/shared/event_watcher_test.go similarity index 94% rename from libraries/shared/watcher_test.go rename to libraries/shared/event_watcher_test.go index 94e8fb33..4a86e5af 100644 --- a/libraries/shared/watcher_test.go +++ b/libraries/shared/event_watcher_test.go @@ -35,12 +35,12 @@ import ( "github.com/vulcanize/vulcanizedb/test_config" ) -var _ = Describe("Watcher", func() { +var _ = Describe("EventWatcher", func() { It("initialises correctly", func() { db := test_config.NewTestDB(core.Node{ID: "testNode"}) bc := fakes.NewMockBlockChain() - watcher := shared.NewWatcher(db, bc) + watcher := shared.NewEventWatcher(db, bc) Expect(watcher.DB).To(Equal(db)) Expect(watcher.Fetcher).NotTo(BeNil()) @@ -48,7 +48,7 @@ var _ = Describe("Watcher", func() { }) It("adds transformers", func() { - watcher := shared.NewWatcher(nil, nil) + watcher := shared.NewEventWatcher(nil, nil) fakeTransformer := &mocks.MockTransformer{} fakeTransformer.SetTransformerConfig(mocks.FakeTransformerConfig) watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformer.FakeTransformerInitializer}) @@ -60,7 +60,7 @@ var _ = Describe("Watcher", func() { }) It("adds transformers from multiple sources", func() { - watcher := shared.NewWatcher(nil, nil) + watcher := shared.NewEventWatcher(nil, nil) fakeTransformer1 := &mocks.MockTransformer{} fakeTransformer1.SetTransformerConfig(mocks.FakeTransformerConfig) @@ -84,7 +84,7 @@ var _ = Describe("Watcher", func() { fakeTransformer2 := &mocks.MockTransformer{} fakeTransformer2.SetTransformerConfig(shared2.TransformerConfig{StartingBlockNumber: 3}) - watcher := shared.NewWatcher(nil, nil) + watcher := shared.NewEventWatcher(nil, nil) watcher.AddTransformers([]shared2.TransformerInitializer{ fakeTransformer1.FakeTransformerInitializer, fakeTransformer2.FakeTransformerInitializer, @@ -94,7 +94,7 @@ var _ = Describe("Watcher", func() { }) It("returns an error when run without transformers", func() { - watcher := shared.NewWatcher(nil, nil) + watcher := shared.NewEventWatcher(nil, nil) err := watcher.Execute() Expect(err).To(MatchError("No transformers added to watcher")) }) @@ -102,7 +102,7 @@ var _ = Describe("Watcher", func() { Describe("with missing headers", func() { var ( db *postgres.DB - watcher shared.Watcher + watcher shared.EventWatcher mockBlockChain fakes.MockBlockChain headerRepository repositories.HeaderRepository repository mocks.MockWatcherRepository @@ -117,7 +117,7 @@ var _ = Describe("Watcher", func() { Expect(err).NotTo(HaveOccurred()) repository = mocks.MockWatcherRepository{} - watcher = shared.NewWatcher(db, &mockBlockChain) + watcher = shared.NewEventWatcher(db, &mockBlockChain) }) It("executes each transformer", func() { @@ -163,7 +163,7 @@ var _ = Describe("Watcher", func() { mockBlockChain.SetGetEthLogsWithCustomQueryReturnLogs([]types.Log{logA, logB}) repository.SetMissingHeaders([]core.Header{fakes.FakeHeader}) - watcher = shared.NewWatcher(db, &mockBlockChain) + watcher = shared.NewEventWatcher(db, &mockBlockChain) watcher.AddTransformers([]shared2.TransformerInitializer{ transformerA.FakeTransformerInitializer, transformerB.FakeTransformerInitializer}) diff --git a/libraries/shared/shared_suite_test.go b/libraries/shared/shared_suite_test.go index 8d079e5d..3e3b5fcc 100644 --- a/libraries/shared/shared_suite_test.go +++ b/libraries/shared/shared_suite_test.go @@ -17,8 +17,8 @@ package shared_test import ( + log "github.com/sirupsen/logrus" "io/ioutil" - "log" "testing" . "github.com/onsi/ginkgo" diff --git a/libraries/shared/storage_watcher.go b/libraries/shared/storage_watcher.go new file mode 100644 index 00000000..50e042b2 --- /dev/null +++ b/libraries/shared/storage_watcher.go @@ -0,0 +1,74 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package shared + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/sirupsen/logrus" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/storage" + "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared" + "strings" + + "github.com/vulcanize/vulcanizedb/pkg/fs" +) + +type StorageWatcher struct { + db *postgres.DB + tailer fs.Tailer + Transformers map[common.Address]storage.Transformer +} + +func NewStorageWatcher(tailer fs.Tailer, db *postgres.DB) StorageWatcher { + transformers := make(map[common.Address]storage.Transformer) + return StorageWatcher{ + db: db, + tailer: tailer, + Transformers: transformers, + } +} + +func (watcher StorageWatcher) AddTransformers(initializers []storage.TransformerInitializer) { + for _, initializer := range initializers { + transformer := initializer(watcher.db) + watcher.Transformers[transformer.ContractAddress()] = transformer + } +} + +func (watcher StorageWatcher) Execute() error { + t, tailErr := watcher.tailer.Tail() + if tailErr != nil { + return tailErr + } + for line := range t.Lines { + row, parseErr := shared.FromStrings(strings.Split(line.Text, ",")) + if parseErr != nil { + return parseErr + } + transformer, ok := watcher.Transformers[row.Contract] + if !ok { + logrus.Warn(shared.ErrContractNotFound{Contract: row.Contract.Hex()}.Error()) + continue + } + executeErr := transformer.Execute(row) + if executeErr != nil { + logrus.Warn(executeErr.Error()) + continue + } + } + return nil +} diff --git a/libraries/shared/storage_watcher_test.go b/libraries/shared/storage_watcher_test.go new file mode 100644 index 00000000..3f7b13d0 --- /dev/null +++ b/libraries/shared/storage_watcher_test.go @@ -0,0 +1,175 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package shared_test + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + "strings" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/hpcloud/tail" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/sirupsen/logrus" + + "github.com/vulcanize/vulcanizedb/libraries/shared" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/fakes" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/storage" + shared2 "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared" + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks" + "github.com/vulcanize/vulcanizedb/test_config" +) + +var _ = Describe("Storage Watcher", func() { + It("adds transformers", func() { + fakeAddress := common.HexToAddress("0x12345") + fakeTransformer := &mocks.MockStorageTransformer{Address: fakeAddress} + watcher := shared.NewStorageWatcher(&fakes.MockTailer{}, test_config.NewTestDB(core.Node{})) + + watcher.AddTransformers([]storage.TransformerInitializer{fakeTransformer.FakeTransformerInitializer}) + + Expect(watcher.Transformers[fakeAddress]).To(Equal(fakeTransformer)) + }) + + It("reads the tail of the storage diffs file", func() { + mockTailer := fakes.NewMockTailer() + watcher := shared.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{})) + + assert(func(err error) { + Expect(err).To(BeNil()) + Expect(mockTailer.TailCalled).To(BeTrue()) + }, watcher, mockTailer, []*tail.Line{}) + }) + + It("returns error if row parsing fails", func() { + mockTailer := fakes.NewMockTailer() + watcher := shared.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{})) + line := &tail.Line{Text: "oops"} + + assert(func(err error) { + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(shared2.ErrRowMalformed{Length: 1})) + }, watcher, mockTailer, []*tail.Line{line}) + }) + + It("logs error if no transformer can parse storage row", func() { + mockTailer := fakes.NewMockTailer() + line := &tail.Line{ + Text: "12345,block_hash,123,storage_key,storage_value", + Time: time.Time{}, + Err: nil, + } + watcher := shared.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{})) + tempFile, err := ioutil.TempFile("", "log") + defer os.Remove(tempFile.Name()) + Expect(err).NotTo(HaveOccurred()) + logrus.SetOutput(tempFile) + + assert(func(err error) { + Expect(err).NotTo(HaveOccurred()) + logContent, readErr := ioutil.ReadFile(tempFile.Name()) + Expect(readErr).NotTo(HaveOccurred()) + Expect(string(logContent)).To(ContainSubstring(shared2.ErrContractNotFound{Contract: common.HexToAddress("0x12345").Hex()}.Error())) + }, watcher, mockTailer, []*tail.Line{line}) + }) + + It("executes transformer with storage row", func() { + address := []byte{1, 2, 3} + blockHash := []byte{4, 5, 6} + blockHeight := int64(789) + storageKey := []byte{9, 8, 7} + storageValue := []byte{6, 5, 4} + mockTailer := fakes.NewMockTailer() + line := &tail.Line{ + Text: fmt.Sprintf("%s,%s,%d,%s,%s", common.Bytes2Hex(address), common.Bytes2Hex(blockHash), blockHeight, common.Bytes2Hex(storageKey), common.Bytes2Hex(storageValue)), + Time: time.Time{}, + Err: nil, + } + watcher := shared.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{})) + fakeTransformer := &mocks.MockStorageTransformer{Address: common.BytesToAddress(address)} + watcher.AddTransformers([]storage.TransformerInitializer{fakeTransformer.FakeTransformerInitializer}) + + assert(func(err error) { + Expect(err).To(BeNil()) + expectedRow, err := shared2.FromStrings(strings.Split(line.Text, ",")) + Expect(err).NotTo(HaveOccurred()) + Expect(fakeTransformer.PassedRow).To(Equal(expectedRow)) + }, watcher, mockTailer, []*tail.Line{line}) + }) + + It("logs error if executing transformer fails", func() { + address := []byte{1, 2, 3} + blockHash := []byte{4, 5, 6} + blockHeight := int64(789) + storageKey := []byte{9, 8, 7} + storageValue := []byte{6, 5, 4} + mockTailer := fakes.NewMockTailer() + line := &tail.Line{ + Text: fmt.Sprintf("%s,%s,%d,%s,%s", common.Bytes2Hex(address), common.Bytes2Hex(blockHash), blockHeight, common.Bytes2Hex(storageKey), common.Bytes2Hex(storageValue)), + Time: time.Time{}, + Err: nil, + } + watcher := shared.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{})) + executionError := errors.New("storage watcher failed attempting to execute transformer") + fakeTransformer := &mocks.MockStorageTransformer{Address: common.BytesToAddress(address), ExecuteErr: executionError} + watcher.AddTransformers([]storage.TransformerInitializer{fakeTransformer.FakeTransformerInitializer}) + tempFile, err := ioutil.TempFile("", "log") + defer os.Remove(tempFile.Name()) + Expect(err).NotTo(HaveOccurred()) + logrus.SetOutput(tempFile) + + assert(func(err error) { + Expect(err).NotTo(HaveOccurred()) + logContent, readErr := ioutil.ReadFile(tempFile.Name()) + Expect(readErr).NotTo(HaveOccurred()) + Expect(string(logContent)).To(ContainSubstring(executionError.Error())) + }, watcher, mockTailer, []*tail.Line{line}) + }) +}) + +func assert(assertion func(err error), watcher shared.StorageWatcher, mockTailer *fakes.MockTailer, lines []*tail.Line) { + errs := make(chan error, 1) + done := make(chan bool, 1) + go execute(watcher, mockTailer, errs, done) + for _, line := range lines { + mockTailer.Lines <- line + } + close(mockTailer.Lines) + + select { + case err := <-errs: + assertion(err) + break + case <-done: + assertion(nil) + break + } +} + +func execute(watcher shared.StorageWatcher, tailer *fakes.MockTailer, errs chan error, done chan bool) { + err := watcher.Execute() + if err != nil { + errs <- err + } else { + done <- true + } +} diff --git a/pkg/datastore/postgres/repositories/header_repository.go b/pkg/datastore/postgres/repositories/header_repository.go index 591496e5..30e4f6e4 100644 --- a/pkg/datastore/postgres/repositories/header_repository.go +++ b/pkg/datastore/postgres/repositories/header_repository.go @@ -52,7 +52,7 @@ func (repository HeaderRepository) CreateOrUpdateHeader(header core.Header) (int func (repository HeaderRepository) GetHeader(blockNumber int64) (core.Header, error) { var header core.Header - err := repository.database.Get(&header, `SELECT block_number, hash, raw FROM headers WHERE block_number = $1 AND eth_node_fingerprint = $2`, + err := repository.database.Get(&header, `SELECT id, block_number, hash, raw, block_timestamp FROM headers WHERE block_number = $1 AND eth_node_fingerprint = $2`, blockNumber, repository.database.Node.ID) return header, err } diff --git a/pkg/datastore/postgres/repositories/header_repository_test.go b/pkg/datastore/postgres/repositories/header_repository_test.go index 8bcd1158..fbcd50bb 100644 --- a/pkg/datastore/postgres/repositories/header_repository_test.go +++ b/pkg/datastore/postgres/repositories/header_repository_test.go @@ -188,9 +188,11 @@ var _ = Describe("Block header repository", func() { dbHeader, err := repo.GetHeader(header.BlockNumber) Expect(err).NotTo(HaveOccurred()) + Expect(dbHeader.Id).NotTo(BeZero()) Expect(dbHeader.BlockNumber).To(Equal(header.BlockNumber)) Expect(dbHeader.Hash).To(Equal(header.Hash)) Expect(dbHeader.Raw).To(MatchJSON(header.Raw)) + Expect(dbHeader.Timestamp).To(Equal(header.Timestamp)) }) It("does not return header for a different node fingerprint", func() { diff --git a/pkg/fakes/mock_header_repository.go b/pkg/fakes/mock_header_repository.go index 1a9bf630..a8a4e6ba 100644 --- a/pkg/fakes/mock_header_repository.go +++ b/pkg/fakes/mock_header_repository.go @@ -27,8 +27,11 @@ type MockHeaderRepository struct { createOrUpdateHeaderErr error createOrUpdateHeaderPassedBlockNumbers []int64 createOrUpdateHeaderReturnID int64 + getHeaderError error + getHeaderReturnBlockHash string missingBlockNumbers []int64 headerExists bool + GetHeaderPassedBlockNumber int64 } func NewMockHeaderRepository() *MockHeaderRepository { @@ -53,8 +56,9 @@ func (repository *MockHeaderRepository) CreateOrUpdateHeader(header core.Header) return repository.createOrUpdateHeaderReturnID, repository.createOrUpdateHeaderErr } -func (*MockHeaderRepository) GetHeader(blockNumber int64) (core.Header, error) { - return core.Header{BlockNumber: blockNumber}, nil +func (repository *MockHeaderRepository) GetHeader(blockNumber int64) (core.Header, error) { + repository.GetHeaderPassedBlockNumber = blockNumber + return core.Header{BlockNumber: blockNumber, Hash: repository.getHeaderReturnBlockHash}, repository.getHeaderError } func (repository *MockHeaderRepository) MissingBlockNumbers(startingBlockNumber, endingBlockNumber int64, nodeID string) ([]int64, error) { @@ -69,6 +73,14 @@ func (repository *MockHeaderRepository) SetHeaderExists(headerExists bool) { repository.headerExists = headerExists } +func (repository *MockHeaderRepository) SetGetHeaderError(err error) { + repository.getHeaderError = err +} + +func (repository *MockHeaderRepository) SetGetHeaderReturnBlockHash(hash string) { + repository.getHeaderReturnBlockHash = hash +} + func (repository *MockHeaderRepository) AssertCreateOrUpdateHeaderCallCountAndPassedBlockNumbers(times int, blockNumbers []int64) { Expect(repository.createOrUpdateHeaderCallCount).To(Equal(times)) Expect(repository.createOrUpdateHeaderPassedBlockNumbers).To(Equal(blockNumbers)) diff --git a/pkg/fakes/mock_tailer.go b/pkg/fakes/mock_tailer.go new file mode 100644 index 00000000..888688d6 --- /dev/null +++ b/pkg/fakes/mock_tailer.go @@ -0,0 +1,29 @@ +package fakes + +import ( + "github.com/hpcloud/tail" + "gopkg.in/tomb.v1" +) + +type MockTailer struct { + Lines chan *tail.Line + TailCalled bool +} + +func NewMockTailer() *MockTailer { + return &MockTailer{ + Lines: make(chan *tail.Line, 1), + TailCalled: false, + } +} + +func (mock *MockTailer) Tail() (*tail.Tail, error) { + mock.TailCalled = true + fakeTail := &tail.Tail{ + Filename: "", + Lines: mock.Lines, + Config: tail.Config{}, + Tomb: tomb.Tomb{}, + } + return fakeTail, nil +} diff --git a/pkg/fs/reader.go b/pkg/fs/reader.go index fe9df6ac..8e886a7d 100644 --- a/pkg/fs/reader.go +++ b/pkg/fs/reader.go @@ -22,8 +22,7 @@ type Reader interface { Read(path string) ([]byte, error) } -type FsReader struct { -} +type FsReader struct{} func (FsReader) Read(path string) ([]byte, error) { return ioutil.ReadFile(path) diff --git a/pkg/fs/tail.go b/pkg/fs/tail.go new file mode 100644 index 00000000..7c84c692 --- /dev/null +++ b/pkg/fs/tail.go @@ -0,0 +1,15 @@ +package fs + +import "github.com/hpcloud/tail" + +type Tailer interface { + Tail() (*tail.Tail, error) +} + +type FileTailer struct { + Path string +} + +func (tailer FileTailer) Tail() (*tail.Tail, error) { + return tail.TailFile(tailer.Path, tail.Config{Follow: true}) +} diff --git a/pkg/transformers/bite/converter.go b/pkg/transformers/bite/converter.go index 6d5b7e2f..556a27dd 100644 --- a/pkg/transformers/bite/converter.go +++ b/pkg/transformers/bite/converter.go @@ -17,7 +17,6 @@ package bite import ( - "bytes" "encoding/json" "fmt" "github.com/ethereum/go-ethereum/accounts/abi/bind" @@ -64,7 +63,7 @@ func (converter BiteConverter) ToModels(entities []interface{}) ([]interface{}, return nil, fmt.Errorf("entity of type %T, not %T", entity, BiteEntity{}) } - ilk := string(bytes.Trim(biteEntity.Ilk[:], "\x00")) + ilk := common.Bytes2Hex(biteEntity.Ilk[:]) urn := common.BytesToAddress(biteEntity.Urn[:]).String() ink := biteEntity.Ink art := biteEntity.Art diff --git a/pkg/transformers/bite/converter_test.go b/pkg/transformers/bite/converter_test.go index 4217defc..0bdbd0db 100644 --- a/pkg/transformers/bite/converter_test.go +++ b/pkg/transformers/bite/converter_test.go @@ -70,7 +70,7 @@ var _ = Describe("Bite Converter", func() { emptyLog, err := json.Marshal(types.Log{}) Expect(err).NotTo(HaveOccurred()) expectedModel := bite.BiteModel{ - Ilk: "", + Ilk: "0000000000000000000000000000000000000000000000000000000000000000", Urn: "0x0000000000000000000000000000000000000000", Ink: "", Art: "", diff --git a/pkg/transformers/cat_file/chop_lump/converter.go b/pkg/transformers/cat_file/chop_lump/converter.go index ca64a0ec..753dd76c 100644 --- a/pkg/transformers/cat_file/chop_lump/converter.go +++ b/pkg/transformers/cat_file/chop_lump/converter.go @@ -40,7 +40,7 @@ func (CatFileChopLumpConverter) ToModels(ethLogs []types.Log) ([]interface{}, er if err != nil { return nil, err } - ilk := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00")) + ilk := shared.GetHexWithoutPrefix(ethLog.Topics[2].Bytes()) what := string(bytes.Trim(ethLog.Topics[3].Bytes(), "\x00")) dataBytes := ethLog.Data[len(ethLog.Data)-constants.DataItemLength:] data := big.NewInt(0).SetBytes(dataBytes).String() diff --git a/pkg/transformers/cat_file/flip/converter.go b/pkg/transformers/cat_file/flip/converter.go index efa8a517..eb748972 100644 --- a/pkg/transformers/cat_file/flip/converter.go +++ b/pkg/transformers/cat_file/flip/converter.go @@ -22,6 +22,7 @@ import ( "errors" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" ) @@ -34,7 +35,7 @@ func (CatFileFlipConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) if err != nil { return nil, err } - ilk := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00")) + ilk := shared.GetHexWithoutPrefix(ethLog.Topics[2].Bytes()) what := string(bytes.Trim(ethLog.Topics[3].Bytes(), "\x00")) flipBytes := ethLog.Data[len(ethLog.Data)-constants.DataItemLength:] flip := common.BytesToAddress(flipBytes).String() diff --git a/pkg/transformers/drip_drip/converter.go b/pkg/transformers/drip_drip/converter.go index a50d9126..534242e1 100644 --- a/pkg/transformers/drip_drip/converter.go +++ b/pkg/transformers/drip_drip/converter.go @@ -17,10 +17,10 @@ package drip_drip import ( - "bytes" "encoding/json" "errors" "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" ) type DripDripConverter struct{} @@ -32,7 +32,7 @@ func (DripDripConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) { if err != nil { return nil, err } - ilk := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00")) + ilk := shared.GetHexWithoutPrefix(ethLog.Topics[2].Bytes()) raw, err := json.Marshal(ethLog) if err != nil { return nil, err diff --git a/pkg/transformers/drip_file/ilk/converter.go b/pkg/transformers/drip_file/ilk/converter.go index adfb2f3d..af155e5b 100644 --- a/pkg/transformers/drip_file/ilk/converter.go +++ b/pkg/transformers/drip_file/ilk/converter.go @@ -38,7 +38,7 @@ func (DripFileIlkConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) return nil, err } - ilk := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00")) + ilk := shared.GetHexWithoutPrefix(ethLog.Topics[2].Bytes()) vow := string(bytes.Trim(ethLog.Topics[3].Bytes(), "\x00")) taxBytes := ethLog.Data[len(ethLog.Data)-constants.DataItemLength:] tax := shared.ConvertToRay(big.NewInt(0).SetBytes(taxBytes).String()) diff --git a/pkg/transformers/factories/storage/storage_suite_test.go b/pkg/transformers/factories/storage/storage_suite_test.go new file mode 100644 index 00000000..b8e3232e --- /dev/null +++ b/pkg/transformers/factories/storage/storage_suite_test.go @@ -0,0 +1,29 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package storage_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestStorage(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Storage Suite") +} diff --git a/pkg/transformers/factories/storage/transformer.go b/pkg/transformers/factories/storage/transformer.go new file mode 100644 index 00000000..68baf5f7 --- /dev/null +++ b/pkg/transformers/factories/storage/transformer.go @@ -0,0 +1,53 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package storage + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/storage" + "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs" + "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared" +) + +type Transformer struct { + Address common.Address + Mappings storage_diffs.Mappings + Repository storage_diffs.Repository +} + +func (transformer Transformer) NewTransformer(db *postgres.DB) storage.Transformer { + transformer.Mappings.SetDB(db) + transformer.Repository.SetDB(db) + return transformer +} + +func (transformer Transformer) ContractAddress() common.Address { + return transformer.Address +} + +func (transformer Transformer) Execute(row shared.StorageDiffRow) error { + metadata, lookupErr := transformer.Mappings.Lookup(row.StorageKey) + if lookupErr != nil { + return lookupErr + } + value, decodeErr := shared.Decode(row, metadata) + if decodeErr != nil { + return decodeErr + } + return transformer.Repository.Create(row.BlockHeight, row.BlockHash.Hex(), metadata, value) +} diff --git a/pkg/transformers/factories/storage/transformer_test.go b/pkg/transformers/factories/storage/transformer_test.go new file mode 100644 index 00000000..43821e46 --- /dev/null +++ b/pkg/transformers/factories/storage/transformer_test.go @@ -0,0 +1,102 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package storage_test + +import ( + "github.com/ethereum/go-ethereum/common" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/fakes" + "github.com/vulcanize/vulcanizedb/pkg/transformers/factories/storage" + "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared" + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks" +) + +var _ = Describe("Storage transformer", func() { + var ( + mappings *mocks.MockMappings + repository *mocks.MockStorageRepository + transformer storage.Transformer + ) + + BeforeEach(func() { + mappings = &mocks.MockMappings{} + repository = &mocks.MockStorageRepository{} + transformer = storage.Transformer{ + Address: common.Address{}, + Mappings: mappings, + Repository: repository, + } + }) + + It("returns the contract address being watched", func() { + fakeAddress := common.HexToAddress("0x12345") + transformer.Address = fakeAddress + + Expect(transformer.ContractAddress()).To(Equal(fakeAddress)) + }) + + It("looks up metadata for storage key", func() { + transformer.Execute(shared.StorageDiffRow{}) + + Expect(mappings.LookupCalled).To(BeTrue()) + }) + + It("returns error if lookup fails", func() { + mappings.LookupErr = fakes.FakeError + + err := transformer.Execute(shared.StorageDiffRow{}) + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + + It("creates storage row with decoded data", func() { + fakeMetadata := shared.StorageValueMetadata{Type: shared.Address} + mappings.Metadata = fakeMetadata + rawValue := common.HexToAddress("0x12345") + fakeBlockNumber := 123 + fakeBlockHash := "0x67890" + fakeRow := shared.StorageDiffRow{ + Contract: common.Address{}, + BlockHash: common.HexToHash(fakeBlockHash), + BlockHeight: fakeBlockNumber, + StorageKey: common.Hash{}, + StorageValue: rawValue.Hash(), + } + + err := transformer.Execute(fakeRow) + + Expect(err).NotTo(HaveOccurred()) + Expect(repository.PassedBlockNumber).To(Equal(fakeBlockNumber)) + Expect(repository.PassedBlockHash).To(Equal(common.HexToHash(fakeBlockHash).Hex())) + Expect(repository.PassedMetadata).To(Equal(fakeMetadata)) + Expect(repository.PassedValue.(string)).To(Equal(rawValue.Hex())) + }) + + It("returns error if creating row fails", func() { + rawValue := common.HexToAddress("0x12345") + fakeMetadata := shared.StorageValueMetadata{Type: shared.Address} + mappings.Metadata = fakeMetadata + repository.CreateErr = fakes.FakeError + + err := transformer.Execute(shared.StorageDiffRow{StorageValue: rawValue.Hash()}) + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) +}) diff --git a/pkg/transformers/frob/converter.go b/pkg/transformers/frob/converter.go index 8b765b91..267626af 100644 --- a/pkg/transformers/frob/converter.go +++ b/pkg/transformers/frob/converter.go @@ -17,7 +17,6 @@ package frob import ( - "bytes" "encoding/json" "fmt" log "github.com/sirupsen/logrus" @@ -68,7 +67,7 @@ func (FrobConverter) ToModels(entities []interface{}) ([]interface{}, error) { return nil, err } model := FrobModel{ - Ilk: string(bytes.Trim(frobEntity.Ilk[:], "\x00)")), + Ilk: common.Bytes2Hex(frobEntity.Ilk[:]), Urn: common.BytesToAddress(frobEntity.Urn[:]).String(), Ink: frobEntity.Ink.String(), Art: frobEntity.Art.String(), diff --git a/pkg/transformers/integration_tests/bite.go b/pkg/transformers/integration_tests/bite.go index 708a98eb..96d7bc5f 100644 --- a/pkg/transformers/integration_tests/bite.go +++ b/pkg/transformers/integration_tests/bite.go @@ -83,7 +83,7 @@ var _ = Describe("Bite Transformer", func() { Expect(len(dbResult)).To(Equal(1)) Expect(dbResult[0].Art).To(Equal("149846666666666655744")) Expect(dbResult[0].IArt).To(Equal("1645356666666666655736")) - Expect(dbResult[0].Ilk).To(Equal("ETH")) + Expect(dbResult[0].Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000")) Expect(dbResult[0].Ink).To(Equal("1000000000000000000")) Expect(dbResult[0].NFlip).To(Equal("2")) Expect(dbResult[0].Tab).To(Equal("149846666666666655744")) diff --git a/pkg/transformers/integration_tests/cat_file.go b/pkg/transformers/integration_tests/cat_file.go index bc9a2358..4b84c94d 100644 --- a/pkg/transformers/integration_tests/cat_file.go +++ b/pkg/transformers/integration_tests/cat_file.go @@ -97,12 +97,12 @@ var _ = Describe("Cat File transformer", func() { Expect(len(dbResult)).To(Equal(2)) sort.Sort(byLogIndexChopLump(dbResult)) - Expect(dbResult[0].Ilk).To(Equal("REP")) + Expect(dbResult[0].Ilk).To(Equal("5245500000000000000000000000000000000000000000000000000000000000")) Expect(dbResult[0].What).To(Equal("lump")) Expect(dbResult[0].Data).To(Equal("10000.000000000000000000")) Expect(dbResult[0].LogIndex).To(Equal(uint(3))) - Expect(dbResult[1].Ilk).To(Equal("REP")) + Expect(dbResult[1].Ilk).To(Equal("5245500000000000000000000000000000000000000000000000000000000000")) Expect(dbResult[1].What).To(Equal("chop")) Expect(dbResult[1].Data).To(Equal("1.000000000000000000000000000")) Expect(dbResult[1].LogIndex).To(Equal(uint(4))) @@ -145,7 +145,7 @@ var _ = Describe("Cat File transformer", func() { Expect(err).NotTo(HaveOccurred()) Expect(len(dbResult)).To(Equal(1)) - Expect(dbResult[0].Ilk).To(Equal("ETH")) + Expect(dbResult[0].Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000")) Expect(dbResult[0].What).To(Equal("flip")) Expect(dbResult[0].Flip).To(Equal("0x32D496Ad866D110060866B7125981C73642cc509")) }) diff --git a/pkg/transformers/integration_tests/drip_drip.go b/pkg/transformers/integration_tests/drip_drip.go index 0ca6e487..385e6bd7 100644 --- a/pkg/transformers/integration_tests/drip_drip.go +++ b/pkg/transformers/integration_tests/drip_drip.go @@ -85,6 +85,6 @@ var _ = Describe("DripDrip Transformer", func() { Expect(len(dbResults)).To(Equal(1)) dbResult := dbResults[0] - Expect(dbResult.Ilk).To(Equal("ETH")) + Expect(dbResult.Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000")) }) }) diff --git a/pkg/transformers/integration_tests/frob.go b/pkg/transformers/integration_tests/frob.go index b3f01742..64de1ac6 100644 --- a/pkg/transformers/integration_tests/frob.go +++ b/pkg/transformers/integration_tests/frob.go @@ -94,7 +94,7 @@ var _ = Describe("Frob Transformer", func() { Expect(dbResult[0].Dart).To(Equal("0")) Expect(dbResult[0].Dink).To(Equal("10000000000000")) Expect(dbResult[0].IArt).To(Equal("1495509999999999999992")) - Expect(dbResult[0].Ilk).To(Equal("ETH")) + Expect(dbResult[0].Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000")) Expect(dbResult[0].Ink).To(Equal("10050100000000000")) Expect(dbResult[0].Urn).To(Equal("0xc8E093e5f3F9B5Aa6A6b33ea45960b93C161430C")) }) diff --git a/pkg/transformers/integration_tests/pit_file_ilk.go b/pkg/transformers/integration_tests/pit_file_ilk.go index 5dfca2e6..85a9be27 100644 --- a/pkg/transformers/integration_tests/pit_file_ilk.go +++ b/pkg/transformers/integration_tests/pit_file_ilk.go @@ -87,7 +87,7 @@ var _ = Describe("PitFileIlk LogNoteTransformer", func() { Expect(err).NotTo(HaveOccurred()) Expect(len(dbResult)).To(Equal(1)) - Expect(dbResult[0].Ilk).To(Equal("ETH")) + Expect(dbResult[0].Ilk).To(Equal("0x4554480000000000000000000000000000000000000000000000000000000000")) Expect(dbResult[0].What).To(Equal("spot")) Expect(dbResult[0].Data).To(Equal("139.840000000000003410605131648")) }) @@ -119,7 +119,7 @@ var _ = Describe("PitFileIlk LogNoteTransformer", func() { pitFileIlkLineModel = result } } - Expect(pitFileIlkLineModel.Ilk).To(Equal("REP")) + Expect(pitFileIlkLineModel.Ilk).To(Equal("0x5245500000000000000000000000000000000000000000000000000000000000")) Expect(pitFileIlkLineModel.Data).To(Equal("2000000.000000000000000000")) }) }) diff --git a/pkg/transformers/integration_tests/vat_flux.go b/pkg/transformers/integration_tests/vat_flux.go index 6f46e2ee..953b6e11 100644 --- a/pkg/transformers/integration_tests/vat_flux.go +++ b/pkg/transformers/integration_tests/vat_flux.go @@ -74,7 +74,7 @@ var _ = Describe("VatFlux LogNoteTransformer", func() { Expect(err).NotTo(HaveOccurred()) Expect(len(dbResult)).To(Equal(1)) - Expect(dbResult[0].Ilk).To(Equal("REP")) + Expect(dbResult[0].Ilk).To(Equal("5245500000000000000000000000000000000000000000000000000000000000")) Expect(dbResult[0].Src).To(Equal("0xC0851F73CC8DD5c0765E71980eC7E7Fd1EF74434")) Expect(dbResult[0].Dst).To(Equal("0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB")) Expect(dbResult[0].Rad).To(Equal("1800000000000000000000000000000000000000000000")) diff --git a/pkg/transformers/integration_tests/vat_fold.go b/pkg/transformers/integration_tests/vat_fold.go index 8357f950..1b1e8113 100644 --- a/pkg/transformers/integration_tests/vat_fold.go +++ b/pkg/transformers/integration_tests/vat_fold.go @@ -82,7 +82,7 @@ var _ = Describe("VatFold Transformer", func() { Expect(len(dbResults)).To(Equal(1)) dbResult := dbResults[0] - Expect(dbResult.Ilk).To(Equal("REP")) + Expect(dbResult.Ilk).To(Equal("5245500000000000000000000000000000000000000000000000000000000000")) Expect(dbResult.Urn).To(Equal(common.HexToAddress("0x0000000000000000000000003728e9777b2a0a611ee0f89e00e01044ce4736d1").String())) Expect(dbResult.Rate).To(Equal("0.000000000000000000000000000")) }) diff --git a/pkg/transformers/integration_tests/vat_grab.go b/pkg/transformers/integration_tests/vat_grab.go index b29f1e7a..caef8338 100644 --- a/pkg/transformers/integration_tests/vat_grab.go +++ b/pkg/transformers/integration_tests/vat_grab.go @@ -75,7 +75,7 @@ var _ = Describe("Vat Grab Transformer", func() { Expect(err).NotTo(HaveOccurred()) Expect(len(dbResult)).To(Equal(1)) - Expect(dbResult[0].Ilk).To(Equal("REP")) + Expect(dbResult[0].Ilk).To(Equal("5245500000000000000000000000000000000000000000000000000000000000")) Expect(dbResult[0].Urn).To(Equal("0x6a3AE20C315E845B2E398e68EfFe39139eC6060C")) Expect(dbResult[0].V).To(Equal("0x2F34f22a00eE4b7a8F8BBC4eAee1658774c624e0")) //cat contract address Expect(dbResult[0].W).To(Equal("0x3728e9777B2a0a611ee0F89e00E01044ce4736d1")) diff --git a/pkg/transformers/integration_tests/vat_init.go b/pkg/transformers/integration_tests/vat_init.go index bbc0e47a..caf390fc 100644 --- a/pkg/transformers/integration_tests/vat_init.go +++ b/pkg/transformers/integration_tests/vat_init.go @@ -74,6 +74,6 @@ var _ = Describe("VatInit LogNoteTransformer", func() { Expect(len(dbResults)).To(Equal(1)) dbResult := dbResults[0] - Expect(dbResult.Ilk).To(Equal("ETH")) + Expect(dbResult.Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000")) }) }) diff --git a/pkg/transformers/integration_tests/vat_slip.go b/pkg/transformers/integration_tests/vat_slip.go index 28a4b160..0cda5d49 100644 --- a/pkg/transformers/integration_tests/vat_slip.go +++ b/pkg/transformers/integration_tests/vat_slip.go @@ -66,7 +66,7 @@ var _ = Describe("Vat slip transformer", func() { var model vat_slip.VatSlipModel err = db.Get(&model, `SELECT ilk, guy, rad, tx_idx FROM maker.vat_slip WHERE header_id = $1`, headerID) Expect(err).NotTo(HaveOccurred()) - Expect(model.Ilk).To(Equal("ETH")) + Expect(model.Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000")) Expect(model.Guy).To(Equal("0xDA15dCE70ab462E66779f23ee14F21d993789eE3")) Expect(model.Rad).To(Equal("100000000000000000000000000000000000000000000000")) Expect(model.TransactionIndex).To(Equal(uint(0))) diff --git a/pkg/transformers/integration_tests/vat_tune.go b/pkg/transformers/integration_tests/vat_tune.go index ba443847..40c8a23a 100644 --- a/pkg/transformers/integration_tests/vat_tune.go +++ b/pkg/transformers/integration_tests/vat_tune.go @@ -75,7 +75,7 @@ var _ = Describe("VatTune LogNoteTransformer", func() { Expect(err).NotTo(HaveOccurred()) Expect(len(dbResult)).To(Equal(1)) - Expect(dbResult[0].Ilk).To(Equal("ETH")) + Expect(dbResult[0].Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000")) Expect(dbResult[0].Urn).To(Equal("0x4F26FfBe5F04ED43630fdC30A87638d53D0b0876")) Expect(dbResult[0].V).To(Equal("0x4F26FfBe5F04ED43630fdC30A87638d53D0b0876")) Expect(dbResult[0].W).To(Equal("0x4F26FfBe5F04ED43630fdC30A87638d53D0b0876")) diff --git a/pkg/transformers/pit_file/ilk/converter.go b/pkg/transformers/pit_file/ilk/converter.go index b1d4a94c..3945f22b 100644 --- a/pkg/transformers/pit_file/ilk/converter.go +++ b/pkg/transformers/pit_file/ilk/converter.go @@ -37,7 +37,7 @@ func (PitFileIlkConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) if err != nil { return nil, err } - ilk := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00")) + ilk := ethLog.Topics[2].Hex() what := string(bytes.Trim(ethLog.Topics[3].Bytes(), "\x00")) dataBytes := ethLog.Data[len(ethLog.Data)-constants.DataItemLength:] data, err := getData(dataBytes, what) diff --git a/pkg/transformers/shared/storage/transformer.go b/pkg/transformers/shared/storage/transformer.go new file mode 100644 index 00000000..240f8ca1 --- /dev/null +++ b/pkg/transformers/shared/storage/transformer.go @@ -0,0 +1,14 @@ +package storage + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared" +) + +type Transformer interface { + Execute(row shared.StorageDiffRow) error + ContractAddress() common.Address +} + +type TransformerInitializer func(db *postgres.DB) Transformer diff --git a/pkg/transformers/shared/utilities.go b/pkg/transformers/shared/utilities.go index 7ee4b5b6..a33587ae 100644 --- a/pkg/transformers/shared/utilities.go +++ b/pkg/transformers/shared/utilities.go @@ -17,6 +17,7 @@ package shared import ( + "github.com/ethereum/go-ethereum/common" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "math/big" ) @@ -59,6 +60,10 @@ func GetDataBytesAtIndex(n int, logData []byte) []byte { return []byte{} } +func GetHexWithoutPrefix(raw []byte) string { + return common.Bytes2Hex(raw) +} + func ConvertToRay(value string) string { return convert(ray, value, rayPrecision) } diff --git a/pkg/transformers/shared/utilities_test.go b/pkg/transformers/shared/utilities_test.go index 905bc1be..56a02e95 100644 --- a/pkg/transformers/shared/utilities_test.go +++ b/pkg/transformers/shared/utilities_test.go @@ -62,4 +62,12 @@ var _ = Describe("Shared utilities", func() { Expect(wadTwo).To(Equal("1.234567890123456690")) }) }) + + Describe("getting hex without prefix", func() { + It("returns bytes as hex without 0x prefix", func() { + raw := common.HexToHash("0x4554480000000000000000000000000000000000000000000000000000000000").Bytes() + result := shared.GetHexWithoutPrefix(raw) + Expect(result).To(Equal("4554480000000000000000000000000000000000000000000000000000000000")) + }) + }) }) diff --git a/pkg/transformers/storage_diffs/maker/maker_storage_repository.go b/pkg/transformers/storage_diffs/maker/maker_storage_repository.go new file mode 100644 index 00000000..5f6cf6ae --- /dev/null +++ b/pkg/transformers/storage_diffs/maker/maker_storage_repository.go @@ -0,0 +1,38 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package maker + +import "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + +type IMakerStorageRepository interface { + GetIlks() ([]string, error) + SetDB(db *postgres.DB) +} + +type MakerStorageRepository struct { + db *postgres.DB +} + +func (repository *MakerStorageRepository) SetDB(db *postgres.DB) { + repository.db = db +} + +func (repository MakerStorageRepository) GetIlks() ([]string, error) { + var ilks []string + err := repository.db.Select(&ilks, `SELECT DISTINCT ilk FROM maker.vat_init`) + return ilks, err +} diff --git a/pkg/transformers/storage_diffs/maker/maker_storage_repository_test.go b/pkg/transformers/storage_diffs/maker/maker_storage_repository_test.go new file mode 100644 index 00000000..262b12b1 --- /dev/null +++ b/pkg/transformers/storage_diffs/maker/maker_storage_repository_test.go @@ -0,0 +1,57 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package maker_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/vulcanize/vulcanizedb/pkg/fakes" + "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/maker" + "github.com/vulcanize/vulcanizedb/test_config" +) + +var _ = Describe("Maker storage repository", func() { + It("fetches unique ilks from vat init events", func() { + db := test_config.NewTestDB(test_config.NewTestNode()) + test_config.CleanTestDB(db) + insertVatInit("ilk1", 1, db) + insertVatInit("ilk2", 2, db) + insertVatInit("ilk2", 3, db) + repository := maker.MakerStorageRepository{} + repository.SetDB(db) + + ilks, err := repository.GetIlks() + + Expect(err).NotTo(HaveOccurred()) + Expect(len(ilks)).To(Equal(2)) + Expect(ilks).To(ConsistOf("ilk1", "ilk2")) + }) +}) + +func insertVatInit(ilk string, blockNumber int64, db *postgres.DB) { + headerRepository := repositories.NewHeaderRepository(db) + headerID, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(blockNumber)) + Expect(err).NotTo(HaveOccurred()) + _, execErr := db.Exec( + `INSERT INTO maker.vat_init (header_id, ilk, log_idx, tx_idx, raw_log) + VALUES($1, $2, $3, $4, $5)`, + headerID, ilk, 0, 0, "[]", + ) + Expect(execErr).NotTo(HaveOccurred()) +} diff --git a/pkg/transformers/storage_diffs/maker/maker_suite_test.go b/pkg/transformers/storage_diffs/maker/maker_suite_test.go new file mode 100644 index 00000000..601507b9 --- /dev/null +++ b/pkg/transformers/storage_diffs/maker/maker_suite_test.go @@ -0,0 +1,29 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package maker_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestMaker(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Maker Suite") +} diff --git a/pkg/transformers/storage_diffs/maker/pit/mappings.go b/pkg/transformers/storage_diffs/maker/pit/mappings.go new file mode 100644 index 00000000..4b9d299a --- /dev/null +++ b/pkg/transformers/storage_diffs/maker/pit/mappings.go @@ -0,0 +1,146 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package pit + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs" + "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/maker" + "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared" + "math/big" +) + +const ( + IlkLine = "line" + IlkSpot = "spot" + PitDrip = "drip" + PitLine = "Line" + PitLive = "live" + PitVat = "vat" +) + +var ( + // storage key and value metadata for "drip" on the Pit contract + DripKey = common.HexToHash(storage_diffs.IndexFive) + DripMetadata = shared.StorageValueMetadata{ + Name: PitDrip, + Key: "", + Type: shared.Address, + } + + IlkSpotIndex = storage_diffs.IndexOne + + // storage key and value metadata for "Spot" on the Pit contract + LineKey = common.HexToHash(storage_diffs.IndexThree) + LineMetadata = shared.StorageValueMetadata{ + Name: PitLine, + Key: "", + Type: shared.Uint256, + } + + // storage key and value metadata for "live" on the Pit contract + LiveKey = common.HexToHash(storage_diffs.IndexTwo) + LiveMetadata = shared.StorageValueMetadata{ + Name: PitLive, + Key: "", + Type: shared.Uint256, + } + + // storage key and value metadata for "vat" on the Pit contract + VatKey = common.HexToHash(storage_diffs.IndexFour) + VatMetadata = shared.StorageValueMetadata{ + Name: PitVat, + Key: "", + Type: shared.Address, + } +) + +type PitMappings struct { + StorageRepository maker.IMakerStorageRepository + mappings map[common.Hash]shared.StorageValueMetadata +} + +func (mappings *PitMappings) SetDB(db *postgres.DB) { + mappings.StorageRepository.SetDB(db) +} + +func (mappings *PitMappings) Lookup(key common.Hash) (shared.StorageValueMetadata, error) { + metadata, ok := mappings.mappings[key] + if !ok { + err := mappings.loadMappings() + if err != nil { + return metadata, err + } + metadata, ok = mappings.mappings[key] + if !ok { + return metadata, shared.ErrStorageKeyNotFound{Key: key.Hex()} + } + } + return metadata, nil +} + +func (mappings *PitMappings) loadMappings() error { + mappings.mappings = getStaticMappings() + ilks, err := mappings.StorageRepository.GetIlks() + if err != nil { + return err + } + for _, ilk := range ilks { + mappings.mappings[getSpotKey(ilk)] = getSpotMetadata(ilk) + mappings.mappings[getLineKey(ilk)] = getLineMetadata(ilk) + } + return nil +} + +func getStaticMappings() map[common.Hash]shared.StorageValueMetadata { + mappings := make(map[common.Hash]shared.StorageValueMetadata) + mappings[DripKey] = DripMetadata + mappings[LineKey] = LineMetadata + mappings[LiveKey] = LiveMetadata + mappings[VatKey] = VatMetadata + return mappings +} + +func getSpotKey(ilk string) common.Hash { + keyBytes := common.FromHex("0x" + ilk + IlkSpotIndex) + encoded := crypto.Keccak256(keyBytes) + return common.BytesToHash(encoded) +} + +func getSpotMetadata(ilk string) shared.StorageValueMetadata { + return shared.StorageValueMetadata{ + Name: IlkSpot, + Key: ilk, + Type: shared.Uint256, + } +} + +func getLineKey(ilk string) common.Hash { + spotMappingAsInt := big.NewInt(0).SetBytes(getSpotKey(ilk).Bytes()) + incrementedByOne := big.NewInt(0).Add(spotMappingAsInt, big.NewInt(1)) + return common.BytesToHash(incrementedByOne.Bytes()) +} + +func getLineMetadata(ilk string) shared.StorageValueMetadata { + return shared.StorageValueMetadata{ + Name: IlkLine, + Key: ilk, + Type: shared.Uint256, + } +} diff --git a/pkg/transformers/storage_diffs/maker/pit/mappings_test.go b/pkg/transformers/storage_diffs/maker/pit/mappings_test.go new file mode 100644 index 00000000..2bbb22aa --- /dev/null +++ b/pkg/transformers/storage_diffs/maker/pit/mappings_test.go @@ -0,0 +1,90 @@ +package pit_test + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/fakes" + "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/maker/pit" + "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/maker/test_helpers" + "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared" + "math/big" +) + +var _ = Describe("Pit storage mappings", func() { + Describe("looking up static keys", func() { + It("returns value metadata if key exists", func() { + storageRepository := &test_helpers.MockMakerStorageRepository{} + mappings := pit.PitMappings{StorageRepository: storageRepository} + + Expect(mappings.Lookup(pit.DripKey)).To(Equal(pit.DripMetadata)) + Expect(mappings.Lookup(pit.LineKey)).To(Equal(pit.LineMetadata)) + Expect(mappings.Lookup(pit.LiveKey)).To(Equal(pit.LiveMetadata)) + Expect(mappings.Lookup(pit.VatKey)).To(Equal(pit.VatMetadata)) + }) + + It("returns error if key does not exist", func() { + mappings := pit.PitMappings{StorageRepository: &test_helpers.MockMakerStorageRepository{}} + + _, err := mappings.Lookup(common.HexToHash(fakes.FakeHash.Hex())) + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(shared.ErrStorageKeyNotFound{Key: fakes.FakeHash.Hex()})) + }) + }) + + Describe("looking up dynamic keys", func() { + It("refreshes mappings from repository if key not found", func() { + storageRepository := &test_helpers.MockMakerStorageRepository{} + mappings := pit.PitMappings{StorageRepository: storageRepository} + + mappings.Lookup(fakes.FakeHash) + + Expect(storageRepository.GetIlksCalled).To(BeTrue()) + }) + + It("returns value metadata for spot when ilk in the DB", func() { + storageRepository := &test_helpers.MockMakerStorageRepository{} + fakeIlk := "fakeIlk" + storageRepository.SetIlks([]string{fakeIlk}) + mappings := pit.PitMappings{StorageRepository: storageRepository} + ilkSpotKey := common.BytesToHash(crypto.Keccak256(common.FromHex("0x" + fakeIlk + pit.IlkSpotIndex))) + expectedMetadata := shared.StorageValueMetadata{ + Name: pit.IlkSpot, + Key: fakeIlk, + Type: shared.Uint256, + } + + Expect(mappings.Lookup(ilkSpotKey)).To(Equal(expectedMetadata)) + }) + + It("returns value metadata for line when ilk in the DB", func() { + storageRepository := &test_helpers.MockMakerStorageRepository{} + fakeIlk := "fakeIlk" + storageRepository.SetIlks([]string{fakeIlk}) + mappings := pit.PitMappings{StorageRepository: storageRepository} + ilkSpotKeyBytes := crypto.Keccak256(common.FromHex("0x" + fakeIlk + pit.IlkSpotIndex)) + ilkSpotAsInt := big.NewInt(0).SetBytes(ilkSpotKeyBytes) + incrementedIlkSpot := big.NewInt(0).Add(ilkSpotAsInt, big.NewInt(1)) + ilkLineKey := common.BytesToHash(incrementedIlkSpot.Bytes()) + expectedMetadata := shared.StorageValueMetadata{ + Name: pit.IlkLine, + Key: fakeIlk, + Type: shared.Uint256, + } + + Expect(mappings.Lookup(ilkLineKey)).To(Equal(expectedMetadata)) + }) + + It("returns error if key not found", func() { + storageRepository := &test_helpers.MockMakerStorageRepository{} + mappings := pit.PitMappings{StorageRepository: storageRepository} + + _, err := mappings.Lookup(fakes.FakeHash) + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(shared.ErrStorageKeyNotFound{Key: fakes.FakeHash.Hex()})) + }) + }) +}) diff --git a/pkg/transformers/storage_diffs/maker/pit/pit_suite_test.go b/pkg/transformers/storage_diffs/maker/pit/pit_suite_test.go new file mode 100644 index 00000000..2f78139f --- /dev/null +++ b/pkg/transformers/storage_diffs/maker/pit/pit_suite_test.go @@ -0,0 +1,13 @@ +package pit_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestPit(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Pit Suite") +} diff --git a/pkg/transformers/storage_diffs/maker/pit/repository.go b/pkg/transformers/storage_diffs/maker/pit/repository.go new file mode 100644 index 00000000..a1c5c9df --- /dev/null +++ b/pkg/transformers/storage_diffs/maker/pit/repository.go @@ -0,0 +1,79 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package pit + +import ( + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared" +) + +type PitStorageRepository struct { + db *postgres.DB +} + +func (repository *PitStorageRepository) SetDB(db *postgres.DB) { + repository.db = db +} + +func (repository PitStorageRepository) Create(blockNumber int, blockHash string, metadata shared.StorageValueMetadata, value interface{}) error { + switch metadata.Name { + case IlkLine: + return repository.insertIlkLine(blockNumber, blockHash, metadata.Key, value.(string)) + case IlkSpot: + return repository.insertIlkSpot(blockNumber, blockHash, metadata.Key, value.(string)) + case PitDrip: + return repository.insertPitDrip(blockNumber, blockHash, value.(string)) + case PitLine: + return repository.insertPitLine(blockNumber, blockHash, value.(string)) + case PitLive: + return repository.insertPitLive(blockNumber, blockHash, value.(string)) + case PitVat: + return repository.insertPitVat(blockNumber, blockHash, value.(string)) + default: + panic("unrecognized storage metadata name") + } +} + +func (repository PitStorageRepository) insertIlkLine(blockNumber int, blockHash string, ilk string, line string) error { + _, err := repository.db.Exec(`INSERT INTO maker.pit_ilk_line (block_number, block_hash, ilk, line) VALUES ($1, $2, $3, $4)`, blockNumber, blockHash, ilk, line) + return err +} + +func (repository PitStorageRepository) insertIlkSpot(blockNumber int, blockHash string, ilk string, spot string) error { + _, err := repository.db.Exec(`INSERT INTO maker.pit_ilk_spot (block_number, block_hash, ilk, spot) VALUES ($1, $2, $3, $4)`, blockNumber, blockHash, ilk, spot) + return err +} + +func (repository PitStorageRepository) insertPitDrip(blockNumber int, blockHash string, drip string) error { + _, err := repository.db.Exec(`INSERT INTO maker.pit_drip (block_number, block_hash, drip) VALUES ($1, $2, $3)`, blockNumber, blockHash, drip) + return err +} + +func (repository PitStorageRepository) insertPitLine(blockNumber int, blockHash string, line string) error { + _, err := repository.db.Exec(`INSERT INTO maker.pit_line (block_number, block_hash, line) VALUES ($1, $2, $3)`, blockNumber, blockHash, line) + return err +} + +func (repository PitStorageRepository) insertPitLive(blockNumber int, blockHash string, live string) error { + _, err := repository.db.Exec(`INSERT INTO maker.pit_live (block_number, block_hash, live) VALUES ($1, $2, $3)`, blockNumber, blockHash, live) + return err +} + +func (repository PitStorageRepository) insertPitVat(blockNumber int, blockHash string, vat string) error { + _, err := repository.db.Exec(`INSERT INTO maker.pit_vat (block_number, block_hash, vat) VALUES ($1, $2, $3)`, blockNumber, blockHash, vat) + return err +} diff --git a/pkg/transformers/storage_diffs/maker/pit/repository_test.go b/pkg/transformers/storage_diffs/maker/pit/repository_test.go new file mode 100644 index 00000000..97f3ac3f --- /dev/null +++ b/pkg/transformers/storage_diffs/maker/pit/repository_test.go @@ -0,0 +1,172 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package pit_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/maker/pit" + "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared" + "github.com/vulcanize/vulcanizedb/test_config" +) + +var _ = Describe("Pit storage repository", func() { + var ( + blockNumber int + blockHash string + db *postgres.DB + err error + repo pit.PitStorageRepository + ) + + BeforeEach(func() { + blockNumber = 123 + blockHash = "expected_block_hash" + db = test_config.NewTestDB(test_config.NewTestNode()) + test_config.CleanTestDB(db) + repo = pit.PitStorageRepository{} + repo.SetDB(db) + }) + + It("persists an ilk line", func() { + expectedIlk := "fake_ilk" + expectedLine := "12345" + ilkLineMetadata := shared.StorageValueMetadata{ + Name: pit.IlkLine, + Key: expectedIlk, + Type: shared.Uint256, + } + err = repo.Create(blockNumber, blockHash, ilkLineMetadata, expectedLine) + + Expect(err).NotTo(HaveOccurred()) + type IlkLine struct { + BlockMetadata + Ilk string + Line string + } + var result IlkLine + err = db.Get(&result, `SELECT block_number, block_hash, ilk, line FROM maker.pit_ilk_line`) + Expect(err).NotTo(HaveOccurred()) + Expect(result.BlockNumber).To(Equal(blockNumber)) + Expect(result.BlockHash).To(Equal(blockHash)) + Expect(result.Ilk).To(Equal(expectedIlk)) + Expect(result.Line).To(Equal(expectedLine)) + }) + + It("persists an ilk spot", func() { + expectedIlk := "fake_ilk" + expectedSpot := "12345" + ilkSpotMetadata := shared.StorageValueMetadata{ + Name: pit.IlkSpot, + Key: expectedIlk, + Type: shared.Uint256, + } + err = repo.Create(blockNumber, blockHash, ilkSpotMetadata, expectedSpot) + + Expect(err).NotTo(HaveOccurred()) + type IlkSpot struct { + BlockMetadata + Ilk string + Spot string + } + var result IlkSpot + err = db.Get(&result, `SELECT block_number, block_hash, ilk, spot FROM maker.pit_ilk_spot`) + Expect(err).NotTo(HaveOccurred()) + Expect(result.BlockNumber).To(Equal(blockNumber)) + Expect(result.BlockHash).To(Equal(blockHash)) + Expect(result.Ilk).To(Equal(expectedIlk)) + Expect(result.Spot).To(Equal(expectedSpot)) + }) + + It("persists a pit drip", func() { + expectedDrip := "0x0123456789abcdef0123" + + err = repo.Create(blockNumber, blockHash, pit.DripMetadata, expectedDrip) + + Expect(err).NotTo(HaveOccurred()) + type PitDrip struct { + BlockMetadata + Drip string + } + var result PitDrip + err = db.Get(&result, `SELECT block_number, block_hash, drip FROM maker.pit_drip`) + Expect(err).NotTo(HaveOccurred()) + Expect(result.BlockNumber).To(Equal(blockNumber)) + Expect(result.BlockHash).To(Equal(blockHash)) + Expect(result.Drip).To(Equal(expectedDrip)) + }) + + It("persists a pit line", func() { + expectedLine := "12345" + + err = repo.Create(blockNumber, blockHash, pit.LineMetadata, expectedLine) + + Expect(err).NotTo(HaveOccurred()) + type PitLine struct { + BlockMetadata + Line string + } + var result PitLine + err = db.Get(&result, `SELECT block_number, block_hash, line FROM maker.pit_line`) + Expect(err).NotTo(HaveOccurred()) + Expect(result.BlockNumber).To(Equal(blockNumber)) + Expect(result.BlockHash).To(Equal(blockHash)) + Expect(result.Line).To(Equal(expectedLine)) + }) + + It("persists a pit live", func() { + expectedLive := "12345" + + err = repo.Create(blockNumber, blockHash, pit.LiveMetadata, expectedLive) + + Expect(err).NotTo(HaveOccurred()) + type PitLive struct { + BlockMetadata + Live string + } + var result PitLive + err = db.Get(&result, `SELECT block_number, block_hash, live FROM maker.pit_live`) + Expect(err).NotTo(HaveOccurred()) + Expect(result.BlockNumber).To(Equal(blockNumber)) + Expect(result.BlockHash).To(Equal(blockHash)) + Expect(result.Live).To(Equal(expectedLive)) + }) + + It("persists a pit vat", func() { + expectedVat := "0x0123456789abcdef0123" + + err = repo.Create(blockNumber, blockHash, pit.VatMetadata, expectedVat) + + Expect(err).NotTo(HaveOccurred()) + type PitVat struct { + BlockMetadata + Vat string + } + var result PitVat + err = db.Get(&result, `SELECT block_number, block_hash, vat FROM maker.pit_vat`) + Expect(err).NotTo(HaveOccurred()) + Expect(result.BlockNumber).To(Equal(blockNumber)) + Expect(result.BlockHash).To(Equal(blockHash)) + Expect(result.Vat).To(Equal(expectedVat)) + }) +}) + +type BlockMetadata struct { + BlockNumber int `db:"block_number"` + BlockHash string `db:"block_hash"` +} diff --git a/pkg/transformers/storage_diffs/maker/test_helpers/maker_storage_repository.go b/pkg/transformers/storage_diffs/maker/test_helpers/maker_storage_repository.go new file mode 100644 index 00000000..6f259df3 --- /dev/null +++ b/pkg/transformers/storage_diffs/maker/test_helpers/maker_storage_repository.go @@ -0,0 +1,19 @@ +package test_helpers + +import "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + +type MockMakerStorageRepository struct { + GetIlksCalled bool + ilks []string +} + +func (repository *MockMakerStorageRepository) GetIlks() ([]string, error) { + repository.GetIlksCalled = true + return repository.ilks, nil +} + +func (repository *MockMakerStorageRepository) SetDB(db *postgres.DB) {} + +func (repository *MockMakerStorageRepository) SetIlks(ilks []string) { + repository.ilks = ilks +} diff --git a/pkg/transformers/storage_diffs/mappings.go b/pkg/transformers/storage_diffs/mappings.go new file mode 100644 index 00000000..6f3cb028 --- /dev/null +++ b/pkg/transformers/storage_diffs/mappings.go @@ -0,0 +1,39 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package storage_diffs + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared" +) + +type Mappings interface { + Lookup(key common.Hash) (shared.StorageValueMetadata, error) + SetDB(db *postgres.DB) +} + +const ( + IndexZero = "0000000000000000000000000000000000000000000000000000000000000000" + IndexOne = "0000000000000000000000000000000000000000000000000000000000000001" + IndexTwo = "0000000000000000000000000000000000000000000000000000000000000002" + IndexThree = "0000000000000000000000000000000000000000000000000000000000000003" + IndexFour = "0000000000000000000000000000000000000000000000000000000000000004" + IndexFive = "0000000000000000000000000000000000000000000000000000000000000005" + IndexSix = "0000000000000000000000000000000000000000000000000000000000000006" + IndexSeven = "0000000000000000000000000000000000000000000000000000000000000007" +) diff --git a/pkg/transformers/storage_diffs/repository.go b/pkg/transformers/storage_diffs/repository.go new file mode 100644 index 00000000..7baedb6a --- /dev/null +++ b/pkg/transformers/storage_diffs/repository.go @@ -0,0 +1,27 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package storage_diffs + +import ( + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared" +) + +type Repository interface { + Create(blockNumber int, blockHash string, metadata shared.StorageValueMetadata, value interface{}) error + SetDB(db *postgres.DB) +} diff --git a/pkg/transformers/storage_diffs/shared/decoder.go b/pkg/transformers/storage_diffs/shared/decoder.go new file mode 100644 index 00000000..753b2416 --- /dev/null +++ b/pkg/transformers/storage_diffs/shared/decoder.go @@ -0,0 +1,42 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package shared + +import ( + "github.com/ethereum/go-ethereum/common" + "math/big" +) + +func Decode(row StorageDiffRow, metadata StorageValueMetadata) (interface{}, error) { + switch metadata.Type { + case Uint256: + return decodeUint256(row.StorageValue.Bytes()), nil + case Address: + return decodeAddress(row.StorageValue.Bytes()), nil + default: + return nil, ErrTypeNotFound{} + } +} + +func decodeUint256(raw []byte) string { + n := big.NewInt(0).SetBytes(raw) + return n.String() +} + +func decodeAddress(raw []byte) string { + return common.BytesToAddress(raw).Hex() +} diff --git a/pkg/transformers/storage_diffs/shared/decoder_test.go b/pkg/transformers/storage_diffs/shared/decoder_test.go new file mode 100644 index 00000000..710e1e16 --- /dev/null +++ b/pkg/transformers/storage_diffs/shared/decoder_test.go @@ -0,0 +1,58 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package shared_test + +import ( + "github.com/ethereum/go-ethereum/common" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared" + "math/big" +) + +var _ = Describe("Storage decoder", func() { + It("decodes uint256", func() { + fakeInt := common.HexToHash("0000000000000000000000000000000000000000000000000000000000000539") + row := shared.StorageDiffRow{StorageValue: fakeInt} + metadata := shared.StorageValueMetadata{Type: shared.Uint256} + + result, err := shared.Decode(row, metadata) + + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(big.NewInt(0).SetBytes(fakeInt.Bytes()).String())) + }) + + It("decodes address", func() { + fakeAddress := common.HexToAddress("0x12345") + row := shared.StorageDiffRow{StorageValue: fakeAddress.Hash()} + metadata := shared.StorageValueMetadata{Type: shared.Address} + + result, err := shared.Decode(row, metadata) + + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(fakeAddress.Hex())) + }) + + It("returns error if attempting to decode unknown type", func() { + metadata := shared.StorageValueMetadata{Type: 100} + + _, err := shared.Decode(shared.StorageDiffRow{}, metadata) + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(shared.ErrTypeNotFound{})) + }) +}) diff --git a/pkg/transformers/storage_diffs/shared/errors.go b/pkg/transformers/storage_diffs/shared/errors.go new file mode 100644 index 00000000..3d74d4a9 --- /dev/null +++ b/pkg/transformers/storage_diffs/shared/errors.go @@ -0,0 +1,61 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package shared + +import "fmt" + +type ErrContractNotFound struct { + Contract string +} + +func (e ErrContractNotFound) Error() string { + return fmt.Sprintf("transformer not found for contract: %s", e.Contract) +} + +type ErrHeaderMismatch struct { + BlockHeight int + DbHash string + DiffHash string +} + +func (e ErrHeaderMismatch) Error() string { + return fmt.Sprintf("header hash in row does not match db at height %d - row: %s, db: %s", e.BlockHeight, e.DbHash, e.DiffHash) +} + +type ErrRowMalformed struct { + Length int +} + +func (e ErrRowMalformed) Error() string { + return fmt.Sprintf("storage row malformed: length %d, expected %d", e.Length, ExpectedRowLength) +} + +type ErrStorageKeyNotFound struct { + Key string +} + +func (e ErrStorageKeyNotFound) Error() string { + return fmt.Sprintf("unknown storage key: %s", e.Key) +} + +type ErrTypeNotFound struct { + Type int +} + +func (e ErrTypeNotFound) Error() string { + return fmt.Sprintf("no decoder for type: %d", e.Type) +} diff --git a/pkg/transformers/storage_diffs/shared/row.go b/pkg/transformers/storage_diffs/shared/row.go new file mode 100644 index 00000000..3de11d41 --- /dev/null +++ b/pkg/transformers/storage_diffs/shared/row.go @@ -0,0 +1,49 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package shared + +import ( + "github.com/ethereum/go-ethereum/common" + "strconv" +) + +const ExpectedRowLength = 5 + +type StorageDiffRow struct { + Contract common.Address + BlockHash common.Hash + BlockHeight int + StorageKey common.Hash + StorageValue common.Hash +} + +func FromStrings(csvRow []string) (StorageDiffRow, error) { + if len(csvRow) != ExpectedRowLength { + return StorageDiffRow{}, ErrRowMalformed{Length: len(csvRow)} + } + height, err := strconv.Atoi(csvRow[2]) + if err != nil { + return StorageDiffRow{}, err + } + return StorageDiffRow{ + Contract: common.HexToAddress(csvRow[0]), + BlockHash: common.HexToHash(csvRow[1]), + BlockHeight: height, + StorageKey: common.HexToHash(csvRow[3]), + StorageValue: common.HexToHash(csvRow[4]), + }, nil +} diff --git a/pkg/transformers/storage_diffs/shared/row_test.go b/pkg/transformers/storage_diffs/shared/row_test.go new file mode 100644 index 00000000..a75201fb --- /dev/null +++ b/pkg/transformers/storage_diffs/shared/row_test.go @@ -0,0 +1,57 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package shared_test + +import ( + "github.com/ethereum/go-ethereum/common" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared" +) + +var _ = Describe("Storage row parsing", func() { + It("converts an array of strings to a row struct", func() { + contract := "0x123" + blockHash := "0x456" + blockHeight := "789" + storageKey := "0x987" + storageValue := "0x654" + data := []string{contract, blockHash, blockHeight, storageKey, storageValue} + + result, err := shared.FromStrings(data) + + Expect(err).NotTo(HaveOccurred()) + Expect(result.Contract).To(Equal(common.HexToAddress(contract))) + Expect(result.BlockHash).To(Equal(common.HexToHash(blockHash))) + Expect(result.BlockHeight).To(Equal(789)) + Expect(result.StorageKey).To(Equal(common.HexToHash(storageKey))) + Expect(result.StorageValue).To(Equal(common.HexToHash(storageValue))) + }) + + It("returns an error if row is missing data", func() { + _, err := shared.FromStrings([]string{"0x123"}) + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(shared.ErrRowMalformed{Length: 1})) + }) + + It("returns error if block height malformed", func() { + _, err := shared.FromStrings([]string{"", "", "", "", ""}) + + Expect(err).To(HaveOccurred()) + }) +}) diff --git a/pkg/transformers/storage_diffs/shared/shared_suite_test.go b/pkg/transformers/storage_diffs/shared/shared_suite_test.go new file mode 100644 index 00000000..a36f747f --- /dev/null +++ b/pkg/transformers/storage_diffs/shared/shared_suite_test.go @@ -0,0 +1,29 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package shared_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestShared(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Shared Suite") +} diff --git a/pkg/transformers/storage_diffs/shared/value.go b/pkg/transformers/storage_diffs/shared/value.go new file mode 100644 index 00000000..d19d95b5 --- /dev/null +++ b/pkg/transformers/storage_diffs/shared/value.go @@ -0,0 +1,31 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package shared + +type ValueType int + +const ( + Uint256 ValueType = iota + Bytes32 + Address +) + +type StorageValueMetadata struct { + Name string + Key string + Type ValueType +} diff --git a/pkg/transformers/storage_transformers.go b/pkg/transformers/storage_transformers.go new file mode 100644 index 00000000..ee811c66 --- /dev/null +++ b/pkg/transformers/storage_transformers.go @@ -0,0 +1,17 @@ +package transformers + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/vulcanize/vulcanizedb/pkg/transformers/factories/storage" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/maker" + "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/maker/pit" +) + +func GetPitStorageTransformer() storage.Transformer { + return storage.Transformer{ + Address: common.HexToAddress(constants.PitContractAddress()), + Mappings: &pit.PitMappings{StorageRepository: &maker.MakerStorageRepository{}}, + Repository: &pit.PitStorageRepository{}, + } +} diff --git a/pkg/transformers/test_data/bite.go b/pkg/transformers/test_data/bite.go index eb6ab6c7..f724d0b5 100644 --- a/pkg/transformers/test_data/bite.go +++ b/pkg/transformers/test_data/bite.go @@ -41,7 +41,7 @@ var ( biteRawJson, _ = json.Marshal(EthBiteLog) biteIlk = [32]byte{69, 84, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} biteLad = [32]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 180, 20, 126, 218, 128, 254, 199, 18, 42, 225, 109, 162, 71, 156, 189, 127, 251} - biteIlkString = "ETH" + biteIlkString = "4554480000000000000000000000000000000000000000000000000000000000" biteLadString = "0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB" ) diff --git a/pkg/transformers/test_data/cat_file.go b/pkg/transformers/test_data/cat_file.go index 6315cfe1..180262a6 100644 --- a/pkg/transformers/test_data/cat_file.go +++ b/pkg/transformers/test_data/cat_file.go @@ -45,7 +45,7 @@ var EthCatFileChopLog = types.Log{ } var rawCatFileChopLog, _ = json.Marshal(EthCatFileChopLog) var CatFileChopModel = chop_lump.CatFileChopLumpModel{ - Ilk: "fake ilk", + Ilk: "66616b6520696c6b000000000000000000000000000000000000000000000000", What: "chop", Data: "123.456789012345680589533003513", TransactionIndex: EthCatFileChopLog.TxIndex, @@ -71,7 +71,7 @@ var EthCatFileLumpLog = types.Log{ } var rawCatFileLumpLog, _ = json.Marshal(EthCatFileLumpLog) var CatFileLumpModel = chop_lump.CatFileChopLumpModel{ - Ilk: "fake ilk", + Ilk: "66616b6520696c6b000000000000000000000000000000000000000000000000", What: "lump", Data: "12345.678901234567092615", TransactionIndex: EthCatFileLumpLog.TxIndex, @@ -98,7 +98,7 @@ var EthCatFileFlipLog = types.Log{ var rawCatFileFlipLog, _ = json.Marshal(EthCatFileFlipLog) var CatFileFlipModel = flip.CatFileFlipModel{ - Ilk: "fake ilk", + Ilk: "66616b6520696c6b000000000000000000000000000000000000000000000000", What: "flip", Flip: "0x07Fa9eF6609cA7921112231F8f195138ebbA2977", TransactionIndex: EthCatFileFlipLog.TxIndex, diff --git a/pkg/transformers/test_data/drip_drip.go b/pkg/transformers/test_data/drip_drip.go index 53754667..f87382e0 100644 --- a/pkg/transformers/test_data/drip_drip.go +++ b/pkg/transformers/test_data/drip_drip.go @@ -44,7 +44,7 @@ var EthDripDripLog = types.Log{ var rawDripDripLog, _ = json.Marshal(EthDripDripLog) var DripDripModel = drip_drip.DripDripModel{ - Ilk: "fake ilk", + Ilk: "66616b6520696c6b000000000000000000000000000000000000000000000000", LogIndex: EthDripDripLog.Index, TransactionIndex: EthDripDripLog.TxIndex, Raw: rawDripDripLog, diff --git a/pkg/transformers/test_data/drip_file.go b/pkg/transformers/test_data/drip_file.go index ba0fe075..8203f07b 100644 --- a/pkg/transformers/test_data/drip_file.go +++ b/pkg/transformers/test_data/drip_file.go @@ -47,7 +47,7 @@ var EthDripFileIlkLog = types.Log{ var rawDripFileIlkLog, _ = json.Marshal(EthDripFileIlkLog) var DripFileIlkModel = ilk2.DripFileIlkModel{ - Ilk: "fake ilk", + Ilk: "66616b6520696c6b000000000000000000000000000000000000000000000000", Vow: "fake vow", Tax: "12300.000000000000000000000000000", LogIndex: EthDripFileIlkLog.Index, diff --git a/pkg/transformers/test_data/frob.go b/pkg/transformers/test_data/frob.go index 7edf187b..f5dc873a 100644 --- a/pkg/transformers/test_data/frob.go +++ b/pkg/transformers/test_data/frob.go @@ -41,7 +41,7 @@ var ( frobLad = [32]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 217, 34, 137, 65, 83, 190, 158, 239, 123, 114, 24, 220, 86, 93, 29, 12, 226, 160, 146} ink = big.NewInt(15) ilk = [32]byte{102, 97, 107, 101, 32, 105, 108, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} - frobIlkString = "fake ilk" + frobIlkString = "66616b6520696c6b000000000000000000000000000000000000000000000000" frobUrnString = "0x64d922894153BE9EEf7b7218dc565d1D0Ce2a092" ) diff --git a/pkg/transformers/test_data/mocks/mappings.go b/pkg/transformers/test_data/mocks/mappings.go new file mode 100644 index 00000000..18ab4a08 --- /dev/null +++ b/pkg/transformers/test_data/mocks/mappings.go @@ -0,0 +1,22 @@ +package mocks + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared" +) + +type MockMappings struct { + Metadata shared.StorageValueMetadata + LookupCalled bool + LookupErr error +} + +func (mappings *MockMappings) Lookup(key common.Hash) (shared.StorageValueMetadata, error) { + mappings.LookupCalled = true + return mappings.Metadata, mappings.LookupErr +} + +func (*MockMappings) SetDB(db *postgres.DB) { + panic("implement me") +} diff --git a/pkg/transformers/test_data/mocks/storage_repository.go b/pkg/transformers/test_data/mocks/storage_repository.go new file mode 100644 index 00000000..62e88186 --- /dev/null +++ b/pkg/transformers/test_data/mocks/storage_repository.go @@ -0,0 +1,26 @@ +package mocks + +import ( + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared" +) + +type MockStorageRepository struct { + CreateErr error + PassedBlockNumber int + PassedBlockHash string + PassedMetadata shared.StorageValueMetadata + PassedValue interface{} +} + +func (repository *MockStorageRepository) Create(blockNumber int, blockHash string, metadata shared.StorageValueMetadata, value interface{}) error { + repository.PassedBlockNumber = blockNumber + repository.PassedBlockHash = blockHash + repository.PassedMetadata = metadata + repository.PassedValue = value + return repository.CreateErr +} + +func (*MockStorageRepository) SetDB(db *postgres.DB) { + panic("implement me") +} diff --git a/pkg/transformers/test_data/mocks/storage_transformer.go b/pkg/transformers/test_data/mocks/storage_transformer.go new file mode 100644 index 00000000..34c00007 --- /dev/null +++ b/pkg/transformers/test_data/mocks/storage_transformer.go @@ -0,0 +1,27 @@ +package mocks + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/storage" + "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared" +) + +type MockStorageTransformer struct { + Address common.Address + ExecuteErr error + PassedRow shared.StorageDiffRow +} + +func (transformer *MockStorageTransformer) Execute(row shared.StorageDiffRow) error { + transformer.PassedRow = row + return transformer.ExecuteErr +} + +func (transformer *MockStorageTransformer) ContractAddress() common.Address { + return transformer.Address +} + +func (transformer *MockStorageTransformer) FakeTransformerInitializer(db *postgres.DB) storage.Transformer { + return transformer +} diff --git a/pkg/transformers/test_data/pit_file.go b/pkg/transformers/test_data/pit_file.go index f798b795..38895da9 100644 --- a/pkg/transformers/test_data/pit_file.go +++ b/pkg/transformers/test_data/pit_file.go @@ -75,7 +75,7 @@ var EthPitFileIlkLineLog = types.Log{ var rawPitFileIlkLineLog, _ = json.Marshal(EthPitFileIlkLineLog) var PitFileIlkLineModel = ilk2.PitFileIlkModel{ - Ilk: "fake ilk", + Ilk: "0x66616b6520696c6b000000000000000000000000000000000000000000000000", What: "line", Data: "0.000001000000000000", LogIndex: EthPitFileIlkLineLog.Index, @@ -102,7 +102,7 @@ var EthPitFileIlkSpotLog = types.Log{ var rawPitFileIlkSpotLog, _ = json.Marshal(EthPitFileIlkSpotLog) var PitFileIlkSpotModel = ilk2.PitFileIlkModel{ - Ilk: "fake ilk", + Ilk: "0x66616b6520696c6b000000000000000000000000000000000000000000000000", What: "spot", Data: "0.000000000000001000000000000", LogIndex: EthPitFileIlkSpotLog.Index, diff --git a/pkg/transformers/test_data/vat_flux.go b/pkg/transformers/test_data/vat_flux.go index 2a5654e7..543364f4 100644 --- a/pkg/transformers/test_data/vat_flux.go +++ b/pkg/transformers/test_data/vat_flux.go @@ -44,7 +44,7 @@ var VatFluxLog = types.Log{ var rawFluxLog, _ = json.Marshal(VatFluxLog) var VatFluxModel = vat_flux.VatFluxModel{ - Ilk: "REP", + Ilk: "5245500000000000000000000000000000000000000000000000000000000000", Src: "0x7FA9EF6609Ca7921112231f8f195138ebba29770", Dst: "0x93086347c52a8878af71bB818509d484c6a2e1bF", Rad: "123", diff --git a/pkg/transformers/test_data/vat_fold.go b/pkg/transformers/test_data/vat_fold.go index be7bf64e..d5357ace 100644 --- a/pkg/transformers/test_data/vat_fold.go +++ b/pkg/transformers/test_data/vat_fold.go @@ -46,7 +46,7 @@ var EthVatFoldLog = types.Log{ var rawVatFoldLog, _ = json.Marshal(EthVatFoldLog) var VatFoldModel = vat_fold.VatFoldModel{ - Ilk: "REP", + Ilk: "5245500000000000000000000000000000000000000000000000000000000000", Urn: "0x3728e9777B2a0a611ee0F89e00E01044ce4736d1", Rate: "0.000000000000000000000000002", LogIndex: EthVatFoldLog.Index, diff --git a/pkg/transformers/test_data/vat_grab.go b/pkg/transformers/test_data/vat_grab.go index f99adf87..5001095a 100644 --- a/pkg/transformers/test_data/vat_grab.go +++ b/pkg/transformers/test_data/vat_grab.go @@ -28,7 +28,7 @@ var EthVatGrabLog = types.Log{ var rawVatGrabLog, _ = json.Marshal(EthVatGrabLog) var VatGrabModel = vat_grab.VatGrabModel{ - Ilk: "REP", + Ilk: "5245500000000000000000000000000000000000000000000000000000000000", Urn: "0x6a3AE20C315E845B2E398e68EfFe39139eC6060C", V: "0x2F34f22a00eE4b7a8F8BBC4eAee1658774c624e0", W: "0x3728e9777B2a0a611ee0F89e00E01044ce4736d1", diff --git a/pkg/transformers/test_data/vat_init.go b/pkg/transformers/test_data/vat_init.go index d26d6311..bc566d24 100644 --- a/pkg/transformers/test_data/vat_init.go +++ b/pkg/transformers/test_data/vat_init.go @@ -46,7 +46,7 @@ var EthVatInitLog = types.Log{ var rawVatInitLog, _ = json.Marshal(EthVatInitLog) var VatInitModel = vat_init.VatInitModel{ - Ilk: "fake ilk", + Ilk: "66616b6520696c6b000000000000000000000000000000000000000000000000", LogIndex: EthVatInitLog.Index, TransactionIndex: EthVatInitLog.TxIndex, Raw: rawVatInitLog, diff --git a/pkg/transformers/test_data/vat_slip.go b/pkg/transformers/test_data/vat_slip.go index 5b5efd9b..60e46a4b 100644 --- a/pkg/transformers/test_data/vat_slip.go +++ b/pkg/transformers/test_data/vat_slip.go @@ -46,7 +46,7 @@ var EthVatSlipLog = types.Log{ var rawVatSlipLog, _ = json.Marshal(EthVatSlipLog) var VatSlipModel = vat_slip.VatSlipModel{ - Ilk: "fake ilk", + Ilk: "66616b6520696c6b000000000000000000000000000000000000000000000000", Guy: common.HexToAddress("0x7d7bEe5fCfD8028cf7b00876C5b1421c800561A6").String(), Rad: "987654321", TransactionIndex: EthVatSlipLog.TxIndex, diff --git a/pkg/transformers/test_data/vat_toll.go b/pkg/transformers/test_data/vat_toll.go index b1aed7fb..e3ee37d1 100644 --- a/pkg/transformers/test_data/vat_toll.go +++ b/pkg/transformers/test_data/vat_toll.go @@ -29,7 +29,7 @@ var EthVatTollLog = types.Log{ var rawVatTollLog, _ = json.Marshal(EthVatTollLog) var VatTollModel = vat_toll.VatTollModel{ - Ilk: "fake ilk", + Ilk: "66616b6520696c6b000000000000000000000000000000000000000000000000", Urn: "0xA3E37186E017747DbA34042e83e3F76Ad3CcE9b0", Take: big.NewInt(123456789).String(), TransactionIndex: EthVatTollLog.TxIndex, diff --git a/pkg/transformers/test_data/vat_tune.go b/pkg/transformers/test_data/vat_tune.go index 596bd7aa..0ad3b250 100644 --- a/pkg/transformers/test_data/vat_tune.go +++ b/pkg/transformers/test_data/vat_tune.go @@ -34,7 +34,7 @@ var dartString = "11579208923731619542357098500868790785326998466564056403945558 var vatTuneDart, _ = new(big.Int).SetString(dartString, 10) var urnAddress = "0x4F26FfBe5F04ED43630fdC30A87638d53D0b0876" var VatTuneModel = vat_tune.VatTuneModel{ - Ilk: "ETH", + Ilk: "4554480000000000000000000000000000000000000000000000000000000000", Urn: urnAddress, V: urnAddress, W: urnAddress, diff --git a/pkg/transformers/vat_flux/converter.go b/pkg/transformers/vat_flux/converter.go index cb2a35eb..a28f78d3 100644 --- a/pkg/transformers/vat_flux/converter.go +++ b/pkg/transformers/vat_flux/converter.go @@ -17,7 +17,6 @@ package vat_flux import ( - "bytes" "encoding/json" "errors" "github.com/ethereum/go-ethereum/common" @@ -36,7 +35,7 @@ func (VatFluxConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) { return nil, err } - ilk := string(bytes.Trim(ethLog.Topics[1].Bytes(), "\x00")) + ilk := shared.GetHexWithoutPrefix(ethLog.Topics[1].Bytes()) src := common.BytesToAddress(ethLog.Topics[2].Bytes()) dst := common.BytesToAddress(ethLog.Topics[3].Bytes()) radBytes := shared.GetDataBytesAtIndex(-1, ethLog.Data) diff --git a/pkg/transformers/vat_fold/converter.go b/pkg/transformers/vat_fold/converter.go index dbfa8a8b..8d0db8a0 100644 --- a/pkg/transformers/vat_fold/converter.go +++ b/pkg/transformers/vat_fold/converter.go @@ -17,7 +17,6 @@ package vat_fold import ( - "bytes" "encoding/json" "errors" "github.com/ethereum/go-ethereum/common" @@ -35,7 +34,7 @@ func (VatFoldConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) { return nil, err } - ilk := string(bytes.Trim(ethLog.Topics[1].Bytes(), "\x00")) + ilk := shared.GetHexWithoutPrefix(ethLog.Topics[1].Bytes()) urn := common.BytesToAddress(ethLog.Topics[2].Bytes()).String() rate := shared.ConvertToRay(ethLog.Topics[3].Big().String()) raw, err := json.Marshal(ethLog) diff --git a/pkg/transformers/vat_grab/converter.go b/pkg/transformers/vat_grab/converter.go index 9ed2d224..0123aa17 100644 --- a/pkg/transformers/vat_grab/converter.go +++ b/pkg/transformers/vat_grab/converter.go @@ -1,7 +1,6 @@ package vat_grab import ( - "bytes" "encoding/json" "errors" "github.com/ethereum/go-ethereum/common" @@ -20,7 +19,7 @@ func (VatGrabConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) { if err != nil { return nil, err } - ilk := string(bytes.Trim(ethLog.Topics[1].Bytes(), "\x00")) + ilk := shared.GetHexWithoutPrefix(ethLog.Topics[1].Bytes()) urn := common.BytesToAddress(ethLog.Topics[2].Bytes()) v := common.BytesToAddress(ethLog.Topics[3].Bytes()) wBytes := shared.GetDataBytesAtIndex(-3, ethLog.Data) diff --git a/pkg/transformers/vat_init/converter.go b/pkg/transformers/vat_init/converter.go index d5efeb16..1b59c2c8 100644 --- a/pkg/transformers/vat_init/converter.go +++ b/pkg/transformers/vat_init/converter.go @@ -17,10 +17,10 @@ package vat_init import ( - "bytes" "encoding/json" "errors" "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" ) type VatInitConverter struct{} @@ -32,7 +32,7 @@ func (VatInitConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) { if err != nil { return nil, err } - ilk := string(bytes.Trim(ethLog.Topics[1].Bytes(), "\x00")) + ilk := shared.GetHexWithoutPrefix(ethLog.Topics[1].Bytes()) raw, err := json.Marshal(ethLog) if err != nil { return nil, err diff --git a/pkg/transformers/vat_slip/converter.go b/pkg/transformers/vat_slip/converter.go index ec380c02..6258100e 100644 --- a/pkg/transformers/vat_slip/converter.go +++ b/pkg/transformers/vat_slip/converter.go @@ -17,11 +17,11 @@ package vat_slip import ( - "bytes" "encoding/json" "errors" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" ) type VatSlipConverter struct{} @@ -33,7 +33,7 @@ func (VatSlipConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) { if err != nil { return nil, err } - ilk := string(bytes.Trim(ethLog.Topics[1].Bytes(), "\x00")) + ilk := shared.GetHexWithoutPrefix(ethLog.Topics[1].Bytes()) guy := common.BytesToAddress(ethLog.Topics[2].Bytes()) rad := ethLog.Topics[3].Big() diff --git a/pkg/transformers/vat_toll/converter.go b/pkg/transformers/vat_toll/converter.go index 7f99bb5c..42bb0b56 100644 --- a/pkg/transformers/vat_toll/converter.go +++ b/pkg/transformers/vat_toll/converter.go @@ -1,9 +1,9 @@ package vat_toll import ( - "bytes" "encoding/json" "errors" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" @@ -18,7 +18,7 @@ func (VatTollConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) { if err != nil { return nil, err } - ilk := string(bytes.Trim(ethLog.Topics[1].Bytes(), "\x00")) + ilk := shared.GetHexWithoutPrefix(ethLog.Topics[1].Bytes()) urn := common.BytesToAddress(ethLog.Topics[2].Bytes()[:common.AddressLength]) take := ethLog.Topics[3].Big() diff --git a/pkg/transformers/vat_tune/converter.go b/pkg/transformers/vat_tune/converter.go index ecb7a233..6df0c1ea 100644 --- a/pkg/transformers/vat_tune/converter.go +++ b/pkg/transformers/vat_tune/converter.go @@ -1,7 +1,6 @@ package vat_tune import ( - "bytes" "encoding/json" "errors" "math/big" @@ -22,7 +21,7 @@ func (VatTuneConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) { if err != nil { return nil, err } - ilk := string(bytes.Trim(ethLog.Topics[1].Bytes(), "\x00")) + ilk := shared.GetHexWithoutPrefix(ethLog.Topics[1].Bytes()) urn := common.BytesToAddress(ethLog.Topics[2].Bytes()) v := common.BytesToAddress(ethLog.Topics[3].Bytes()) wBytes := shared.GetDataBytesAtIndex(-3, ethLog.Data)