cerc-io/ipld-eth-server
17
0
Fork 1
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases 37 Wiki Activity

(VDB-298) Consume Pit contract storage diffs

Browse Source 
- Continuously parse storage diffs CSV data to read Pit contract state
- Convert ilks in database to raw bytes32 value for use in generating
  storage keys dynamically
- Persist storage diffs with block number and hash for validation
This commit is contained in:
Rob Mulholand 2019-01-28 16:52:47 -06:00
parent c474933432
commit 867f92c431
85 changed files with 2143 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/backfillMakerLogs.go
View File

@ -47,7 +47,7 @@ func backfillMakerLogs() {
log.Fatal("Failed to initialize database.")
}
watcher := shared.NewWatcher(db, blockChain)
watcher := shared.NewEventWatcher(db, blockChain)
watcher.AddTransformers(transformers.TransformerInitializers())
err = watcher.Execute()

2
cmd/continuousLogSync.go
View File

@ -62,7 +62,7 @@ func syncMakerLogs() {
initializers := getTransformerInitializers(transformerNames)
watcher := shared.NewWatcher(db, blockChain)
watcher := shared.NewEventWatcher(db, blockChain)
watcher.AddTransformers(initializers)
for range ticker.C {

66
cmd/parseStorageDiffs.go Normal file
View File

@ -0,0 +1,66 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"github.com/spf13/cobra"
"github.com/vulcanize/vulcanizedb/libraries/shared"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/fs"
"github.com/vulcanize/vulcanizedb/pkg/transformers"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/storage"
"log"
)
// parseStorageDiffsCmd represents the parseStorageDiffs command
var parseStorageDiffsCmd = &cobra.Command{
Use: "parseStorageDiffs",
Short: "Continuously ingest storage diffs from a CSV file",
Long: `Read storage diffs out of a CSV file that is constantly receiving
new rows from an Ethereum node. For example:
./vulcanizedb parseStorageDiffs --config=environments/staging.toml
Note that the path to your storage diffs must be configured in your toml
file under storageDiffsPath.`,
Run: func(cmd *cobra.Command, args []string) {
parseStorageDiffs()
},
}
func init() {
rootCmd.AddCommand(parseStorageDiffsCmd)
}
func parseStorageDiffs() {
blockChain := getBlockChain()
db, err := postgres.NewDB(databaseConfig, blockChain.Node())
if err != nil {
log.Fatal("Failed to initialize database: ", err)
}
tailer := fs.FileTailer{Path: storageDiffsPath}
// TODO: configure transformers
watcher := shared.NewStorageWatcher(tailer, db)
watcher.AddTransformers([]storage.TransformerInitializer{transformers.GetPitStorageTransformer().NewTransformer})
err = watcher.Execute()
if err != nil {
log.Fatal(err)
}
}

4
cmd/root.go
View File

@ -40,6 +40,7 @@ var (
ipc string
levelDbPath string
startingBlockNumber int64
storageDiffsPath string
syncAll bool
endingBlockNumber int64
)
@ -64,6 +65,7 @@ func Execute() {
func database(cmd *cobra.Command, args []string) {
ipc = viper.GetString("client.ipcpath")
levelDbPath = viper.GetString("client.leveldbpath")
storageDiffsPath = viper.GetString("filesystem.storageDiffsPath")
databaseConfig = config.Database{
Name: viper.GetString("database.name"),
Hostname: viper.GetString("database.hostname"),
@ -86,6 +88,7 @@ func init() {
rootCmd.PersistentFlags().String("client-ipcPath", "", "location of geth.ipc file")
rootCmd.PersistentFlags().String("client-levelDbPath", "", "location of levelDb chaindata")
rootCmd.PersistentFlags().String("datadog-name", "vulcanize-test", "datadog service name")
rootCmd.PersistentFlags().String("filesystem-storageDiffsPath", "", "location of storage diffs csv file")
viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name"))
viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port"))
@ -95,6 +98,7 @@ func init() {
viper.BindPFlag("client.ipcPath", rootCmd.PersistentFlags().Lookup("client-ipcPath"))
viper.BindPFlag("client.levelDbPath", rootCmd.PersistentFlags().Lookup("client-levelDbPath"))
viper.BindPFlag("datadog.name", rootCmd.PersistentFlags().Lookup("datadog-name"))
viper.BindPFlag("filesystem.storageDiffsPath", rootCmd.PersistentFlags().Lookup("filesystem-storageDiffsPath"))
}
func initConfig() {

52
db/migrations/00071_create_pit_contract_storage_tables.sql Normal file
View File

@ -0,0 +1,52 @@
-- +goose Up
CREATE TABLE maker.pit_drip (
id SERIAL PRIMARY KEY,
block_number BIGINT,
block_hash TEXT,
drip TEXT
);
CREATE TABLE maker.pit_ilk_spot (
id SERIAL PRIMARY KEY,
block_number BIGINT,
block_hash TEXT,
ilk TEXT,
spot NUMERIC NOT NULL
);
CREATE TABLE maker.pit_ilk_line (
id SERIAL PRIMARY KEY,
block_number BIGINT,
block_hash TEXT,
ilk TEXT,
line NUMERIC NOT NULL
);
CREATE TABLE maker.pit_line (
id SERIAL PRIMARY KEY,
block_number BIGINT,
block_hash TEXT,
line NUMERIC NOT NULL
);
CREATE TABLE maker.pit_live (
id SERIAL PRIMARY KEY,
block_number BIGINT,
block_hash TEXT,
live NUMERIC NOT NULL
);
CREATE TABLE maker.pit_vat (
id SERIAL PRIMARY KEY,
block_number BIGINT,
block_hash TEXT,
vat TEXT
);
-- +goose Down
DROP TABLE maker.pit_drip;
DROP TABLE maker.pit_ilk_spot;
DROP TABLE maker.pit_ilk_line;
DROP TABLE maker.pit_line;
DROP TABLE maker.pit_live;
DROP TABLE maker.pit_vat;

303
db/schema.sql
View File

@ -3,7 +3,7 @@
--
-- Dumped from database version 10.5
-- Dumped by pg_dump version 11.1
-- Dumped by pg_dump version 10.5
SET statement_timeout = 0;
SET lock_timeout = 0;
@ -22,6 +22,20 @@ SET row_security = off;
CREATE SCHEMA maker;
--
-- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: -
--
CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog;
--
-- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: -
--
COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language';
--
-- Name: notify_pricefeed(); Type: FUNCTION; Schema: public; Owner: -
--
@ -559,6 +573,38 @@ CREATE SEQUENCE maker.frob_id_seq
ALTER SEQUENCE maker.frob_id_seq OWNED BY maker.frob.id;
--
-- Name: pit_drip; Type: TABLE; Schema: maker; Owner: -
--
CREATE TABLE maker.pit_drip (
id integer NOT NULL,
block_number bigint,
block_hash text,
drip text
);
--
-- Name: pit_drip_id_seq; Type: SEQUENCE; Schema: maker; Owner: -
--
CREATE SEQUENCE maker.pit_drip_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: pit_drip_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: -
--
ALTER SEQUENCE maker.pit_drip_id_seq OWNED BY maker.pit_drip.id;
--
-- Name: pit_file_debt_ceiling; Type: TABLE; Schema: maker; Owner: -
--
@ -630,6 +676,168 @@ CREATE SEQUENCE maker.pit_file_ilk_id_seq
ALTER SEQUENCE maker.pit_file_ilk_id_seq OWNED BY maker.pit_file_ilk.id;
--
-- Name: pit_ilk_line; Type: TABLE; Schema: maker; Owner: -
--
CREATE TABLE maker.pit_ilk_line (
id integer NOT NULL,
block_number bigint,
block_hash text,
ilk text,
line numeric NOT NULL
);
--
-- Name: pit_ilk_line_id_seq; Type: SEQUENCE; Schema: maker; Owner: -
--
CREATE SEQUENCE maker.pit_ilk_line_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: pit_ilk_line_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: -
--
ALTER SEQUENCE maker.pit_ilk_line_id_seq OWNED BY maker.pit_ilk_line.id;
--
-- Name: pit_ilk_spot; Type: TABLE; Schema: maker; Owner: -
--
CREATE TABLE maker.pit_ilk_spot (
id integer NOT NULL,
block_number bigint,
block_hash text,
ilk text,
spot numeric NOT NULL
);
--
-- Name: pit_ilk_spot_id_seq; Type: SEQUENCE; Schema: maker; Owner: -
--
CREATE SEQUENCE maker.pit_ilk_spot_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: pit_ilk_spot_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: -
--
ALTER SEQUENCE maker.pit_ilk_spot_id_seq OWNED BY maker.pit_ilk_spot.id;
--
-- Name: pit_line; Type: TABLE; Schema: maker; Owner: -
--
CREATE TABLE maker.pit_line (
id integer NOT NULL,
block_number bigint,
block_hash text,
line numeric NOT NULL
);
--
-- Name: pit_line_id_seq; Type: SEQUENCE; Schema: maker; Owner: -
--
CREATE SEQUENCE maker.pit_line_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: pit_line_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: -
--
ALTER SEQUENCE maker.pit_line_id_seq OWNED BY maker.pit_line.id;
--
-- Name: pit_live; Type: TABLE; Schema: maker; Owner: -
--
CREATE TABLE maker.pit_live (
id integer NOT NULL,
block_number bigint,
block_hash text,
live numeric NOT NULL
);
--
-- Name: pit_live_id_seq; Type: SEQUENCE; Schema: maker; Owner: -
--
CREATE SEQUENCE maker.pit_live_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: pit_live_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: -
--
ALTER SEQUENCE maker.pit_live_id_seq OWNED BY maker.pit_live.id;
--
-- Name: pit_vat; Type: TABLE; Schema: maker; Owner: -
--
CREATE TABLE maker.pit_vat (
id integer NOT NULL,
block_number bigint,
block_hash text,
vat text
);
--
-- Name: pit_vat_id_seq; Type: SEQUENCE; Schema: maker; Owner: -
--
CREATE SEQUENCE maker.pit_vat_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: pit_vat_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: -
--
ALTER SEQUENCE maker.pit_vat_id_seq OWNED BY maker.pit_vat.id;
--
-- Name: price_feeds; Type: TABLE; Schema: maker; Owner: -
--
@ -1618,6 +1826,13 @@ ALTER TABLE ONLY maker.flop_kick ALTER COLUMN id SET DEFAULT nextval('maker.flop
ALTER TABLE ONLY maker.frob ALTER COLUMN id SET DEFAULT nextval('maker.frob_id_seq'::regclass);
--
-- Name: pit_drip id; Type: DEFAULT; Schema: maker; Owner: -
--
ALTER TABLE ONLY maker.pit_drip ALTER COLUMN id SET DEFAULT nextval('maker.pit_drip_id_seq'::regclass);
--
-- Name: pit_file_debt_ceiling id; Type: DEFAULT; Schema: maker; Owner: -
--
@ -1632,6 +1847,41 @@ ALTER TABLE ONLY maker.pit_file_debt_ceiling ALTER COLUMN id SET DEFAULT nextval
ALTER TABLE ONLY maker.pit_file_ilk ALTER COLUMN id SET DEFAULT nextval('maker.pit_file_ilk_id_seq'::regclass);
--
-- Name: pit_ilk_line id; Type: DEFAULT; Schema: maker; Owner: -
--
ALTER TABLE ONLY maker.pit_ilk_line ALTER COLUMN id SET DEFAULT nextval('maker.pit_ilk_line_id_seq'::regclass);
--
-- Name: pit_ilk_spot id; Type: DEFAULT; Schema: maker; Owner: -
--
ALTER TABLE ONLY maker.pit_ilk_spot ALTER COLUMN id SET DEFAULT nextval('maker.pit_ilk_spot_id_seq'::regclass);
--
-- Name: pit_line id; Type: DEFAULT; Schema: maker; Owner: -
--
ALTER TABLE ONLY maker.pit_line ALTER COLUMN id SET DEFAULT nextval('maker.pit_line_id_seq'::regclass);
--
-- Name: pit_live id; Type: DEFAULT; Schema: maker; Owner: -
--
ALTER TABLE ONLY maker.pit_live ALTER COLUMN id SET DEFAULT nextval('maker.pit_live_id_seq'::regclass);
--
-- Name: pit_vat id; Type: DEFAULT; Schema: maker; Owner: -
--
ALTER TABLE ONLY maker.pit_vat ALTER COLUMN id SET DEFAULT nextval('maker.pit_vat_id_seq'::regclass);
--
-- Name: price_feeds id; Type: DEFAULT; Schema: maker; Owner: -
--
@ -2017,6 +2267,14 @@ ALTER TABLE ONLY maker.frob
ADD CONSTRAINT frob_pkey PRIMARY KEY (id);
--
-- Name: pit_drip pit_drip_pkey; Type: CONSTRAINT; Schema: maker; Owner: -
--
ALTER TABLE ONLY maker.pit_drip
ADD CONSTRAINT pit_drip_pkey PRIMARY KEY (id);
--
-- Name: pit_file_debt_ceiling pit_file_debt_ceiling_header_id_tx_idx_log_idx_key; Type: CONSTRAINT; Schema: maker; Owner: -
--
@ -2049,6 +2307,46 @@ ALTER TABLE ONLY maker.pit_file_ilk
ADD CONSTRAINT pit_file_ilk_pkey PRIMARY KEY (id);
--
-- Name: pit_ilk_line pit_ilk_line_pkey; Type: CONSTRAINT; Schema: maker; Owner: -
--
ALTER TABLE ONLY maker.pit_ilk_line
ADD CONSTRAINT pit_ilk_line_pkey PRIMARY KEY (id);
--
-- Name: pit_ilk_spot pit_ilk_spot_pkey; Type: CONSTRAINT; Schema: maker; Owner: -
--
ALTER TABLE ONLY maker.pit_ilk_spot
ADD CONSTRAINT pit_ilk_spot_pkey PRIMARY KEY (id);
--
-- Name: pit_line pit_line_pkey; Type: CONSTRAINT; Schema: maker; Owner: -
--
ALTER TABLE ONLY maker.pit_line
ADD CONSTRAINT pit_line_pkey PRIMARY KEY (id);
--
-- Name: pit_live pit_live_pkey; Type: CONSTRAINT; Schema: maker; Owner: -
--
ALTER TABLE ONLY maker.pit_live
ADD CONSTRAINT pit_live_pkey PRIMARY KEY (id);
--
-- Name: pit_vat pit_vat_pkey; Type: CONSTRAINT; Schema: maker; Owner: -
--
ALTER TABLE ONLY maker.pit_vat
ADD CONSTRAINT pit_vat_pkey PRIMARY KEY (id);
--
-- Name: price_feeds price_feeds_header_id_medianizer_address_tx_idx_log_idx_key; Type: CONSTRAINT; Schema: maker; Owner: -
--
@ -2677,4 +2975,5 @@ ALTER TABLE ONLY public.logs
--
-- PostgreSQL database dump complete
--
--

3
environments/staging.toml
View File

@ -11,6 +11,9 @@
[datadog]
name = "maker_vdb_staging"
[filesystem]
storageDiffsPath = "INSERT-PATH-TO-STORAGE-DIFFS"
[contract]
[contract.address]
cat = "0x2f34f22a00ee4b7a8f8bbc4eaee1658774c624e0"

10
libraries/shared/watcher.go → libraries/shared/event_watcher.go
View File

@ -25,7 +25,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
)
type Watcher struct {
type EventWatcher struct {
Transformers []shared.Transformer
DB *postgres.DB
Fetcher shared.LogFetcher
@ -35,10 +35,10 @@ type Watcher struct {
StartingBlock *int64
}
func NewWatcher(db *postgres.DB, bc core.BlockChain) Watcher {
func NewEventWatcher(db *postgres.DB, bc core.BlockChain) EventWatcher {
chunker := shared.NewLogChunker()
fetcher := shared.NewFetcher(bc)
return Watcher{
return EventWatcher{
DB: db,
Fetcher: fetcher,
Chunker: chunker,
@ -46,7 +46,7 @@ func NewWatcher(db *postgres.DB, bc core.BlockChain) Watcher {
}
// Adds transformers to the watcher and updates the chunker, so that it will consider the new transformers.
func (watcher *Watcher) AddTransformers(initializers []shared.TransformerInitializer) {
func (watcher *EventWatcher) AddTransformers(initializers []shared.TransformerInitializer) {
var contractAddresses []common.Address
var topic0s []common.Hash
var configs []shared.TransformerConfig
@ -74,7 +74,7 @@ func (watcher *Watcher) AddTransformers(initializers []shared.TransformerInitial
watcher.Chunker.AddConfigs(configs)
}
func (watcher *Watcher) Execute() error {
func (watcher *EventWatcher) Execute() error {
if watcher.Transformers == nil {
return fmt.Errorf("No transformers added to watcher")
}

18
libraries/shared/watcher_test.go → libraries/shared/event_watcher_test.go
View File

@ -35,12 +35,12 @@ import (
"github.com/vulcanize/vulcanizedb/test_config"
)
var _ = Describe("Watcher", func() {
var _ = Describe("EventWatcher", func() {
It("initialises correctly", func() {
db := test_config.NewTestDB(core.Node{ID: "testNode"})
bc := fakes.NewMockBlockChain()
watcher := shared.NewWatcher(db, bc)
watcher := shared.NewEventWatcher(db, bc)
Expect(watcher.DB).To(Equal(db))
Expect(watcher.Fetcher).NotTo(BeNil())
@ -48,7 +48,7 @@ var _ = Describe("Watcher", func() {
})
It("adds transformers", func() {
watcher := shared.NewWatcher(nil, nil)
watcher := shared.NewEventWatcher(nil, nil)
fakeTransformer := &mocks.MockTransformer{}
fakeTransformer.SetTransformerConfig(mocks.FakeTransformerConfig)
watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformer.FakeTransformerInitializer})
@ -60,7 +60,7 @@ var _ = Describe("Watcher", func() {
})
It("adds transformers from multiple sources", func() {
watcher := shared.NewWatcher(nil, nil)
watcher := shared.NewEventWatcher(nil, nil)
fakeTransformer1 := &mocks.MockTransformer{}
fakeTransformer1.SetTransformerConfig(mocks.FakeTransformerConfig)
@ -84,7 +84,7 @@ var _ = Describe("Watcher", func() {
fakeTransformer2 := &mocks.MockTransformer{}
fakeTransformer2.SetTransformerConfig(shared2.TransformerConfig{StartingBlockNumber: 3})
watcher := shared.NewWatcher(nil, nil)
watcher := shared.NewEventWatcher(nil, nil)
watcher.AddTransformers([]shared2.TransformerInitializer{
fakeTransformer1.FakeTransformerInitializer,
fakeTransformer2.FakeTransformerInitializer,
@ -94,7 +94,7 @@ var _ = Describe("Watcher", func() {
})
It("returns an error when run without transformers", func() {
watcher := shared.NewWatcher(nil, nil)
watcher := shared.NewEventWatcher(nil, nil)
err := watcher.Execute()
Expect(err).To(MatchError("No transformers added to watcher"))
})
@ -102,7 +102,7 @@ var _ = Describe("Watcher", func() {
Describe("with missing headers", func() {
var (
db *postgres.DB
watcher shared.Watcher
watcher shared.EventWatcher
mockBlockChain fakes.MockBlockChain
headerRepository repositories.HeaderRepository
repository mocks.MockWatcherRepository
@ -117,7 +117,7 @@ var _ = Describe("Watcher", func() {
Expect(err).NotTo(HaveOccurred())
repository = mocks.MockWatcherRepository{}
watcher = shared.NewWatcher(db, &mockBlockChain)
watcher = shared.NewEventWatcher(db, &mockBlockChain)
})
It("executes each transformer", func() {
@ -163,7 +163,7 @@ var _ = Describe("Watcher", func() {
mockBlockChain.SetGetEthLogsWithCustomQueryReturnLogs([]types.Log{logA, logB})
repository.SetMissingHeaders([]core.Header{fakes.FakeHeader})
watcher = shared.NewWatcher(db, &mockBlockChain)
watcher = shared.NewEventWatcher(db, &mockBlockChain)
watcher.AddTransformers([]shared2.TransformerInitializer{
transformerA.FakeTransformerInitializer, transformerB.FakeTransformerInitializer})

2
libraries/shared/shared_suite_test.go
View File

@ -17,8 +17,8 @@
package shared_test
import (
log "github.com/sirupsen/logrus"
"io/ioutil"
"log"
"testing"
. "github.com/onsi/ginkgo"

74
libraries/shared/storage_watcher.go Normal file
View File

@ -0,0 +1,74 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package shared
import (
"github.com/ethereum/go-ethereum/common"
"github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/storage"
"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
"strings"
"github.com/vulcanize/vulcanizedb/pkg/fs"
)
type StorageWatcher struct {
db *postgres.DB
tailer fs.Tailer
Transformers map[common.Address]storage.Transformer
}
func NewStorageWatcher(tailer fs.Tailer, db *postgres.DB) StorageWatcher {
transformers := make(map[common.Address]storage.Transformer)
return StorageWatcher{
db: db,
tailer: tailer,
Transformers: transformers,
}
}
func (watcher StorageWatcher) AddTransformers(initializers []storage.TransformerInitializer) {
for _, initializer := range initializers {
transformer := initializer(watcher.db)
watcher.Transformers[transformer.ContractAddress()] = transformer
}
}
func (watcher StorageWatcher) Execute() error {
t, tailErr := watcher.tailer.Tail()
if tailErr != nil {
return tailErr
}
for line := range t.Lines {
row, parseErr := shared.FromStrings(strings.Split(line.Text, ","))
if parseErr != nil {
return parseErr
}
transformer, ok := watcher.Transformers[row.Contract]
if !ok {
logrus.Warn(shared.ErrContractNotFound{Contract: row.Contract.Hex()}.Error())
continue
}
executeErr := transformer.Execute(row)
if executeErr != nil {
logrus.Warn(executeErr.Error())
continue
}
}
return nil
}

175
libraries/shared/storage_watcher_test.go Normal file
View File

@ -0,0 +1,175 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package shared_test
import (
"errors"
"fmt"
"io/ioutil"
"os"
"strings"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/hpcloud/tail"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/libraries/shared"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/storage"
shared2 "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks"
"github.com/vulcanize/vulcanizedb/test_config"
)
var _ = Describe("Storage Watcher", func() {
It("adds transformers", func() {
fakeAddress := common.HexToAddress("0x12345")
fakeTransformer := &mocks.MockStorageTransformer{Address: fakeAddress}
watcher := shared.NewStorageWatcher(&fakes.MockTailer{}, test_config.NewTestDB(core.Node{}))
watcher.AddTransformers([]storage.TransformerInitializer{fakeTransformer.FakeTransformerInitializer})
Expect(watcher.Transformers[fakeAddress]).To(Equal(fakeTransformer))
})
It("reads the tail of the storage diffs file", func() {
mockTailer := fakes.NewMockTailer()
watcher := shared.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
assert(func(err error) {
Expect(err).To(BeNil())
Expect(mockTailer.TailCalled).To(BeTrue())
}, watcher, mockTailer, []*tail.Line{})
})
It("returns error if row parsing fails", func() {
mockTailer := fakes.NewMockTailer()
watcher := shared.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
line := &tail.Line{Text: "oops"}
assert(func(err error) {
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(shared2.ErrRowMalformed{Length: 1}))
}, watcher, mockTailer, []*tail.Line{line})
})
It("logs error if no transformer can parse storage row", func() {
mockTailer := fakes.NewMockTailer()
line := &tail.Line{
Text: "12345,block_hash,123,storage_key,storage_value",
Time: time.Time{},
Err: nil,
}
watcher := shared.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
tempFile, err := ioutil.TempFile("", "log")
defer os.Remove(tempFile.Name())
Expect(err).NotTo(HaveOccurred())
logrus.SetOutput(tempFile)
assert(func(err error) {
Expect(err).NotTo(HaveOccurred())
logContent, readErr := ioutil.ReadFile(tempFile.Name())
Expect(readErr).NotTo(HaveOccurred())
Expect(string(logContent)).To(ContainSubstring(shared2.ErrContractNotFound{Contract: common.HexToAddress("0x12345").Hex()}.Error()))
}, watcher, mockTailer, []*tail.Line{line})
})
It("executes transformer with storage row", func() {
address := []byte{1, 2, 3}
blockHash := []byte{4, 5, 6}
blockHeight := int64(789)
storageKey := []byte{9, 8, 7}
storageValue := []byte{6, 5, 4}
mockTailer := fakes.NewMockTailer()
line := &tail.Line{
Text: fmt.Sprintf("%s,%s,%d,%s,%s", common.Bytes2Hex(address), common.Bytes2Hex(blockHash), blockHeight, common.Bytes2Hex(storageKey), common.Bytes2Hex(storageValue)),
Time: time.Time{},
Err: nil,
}
watcher := shared.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
fakeTransformer := &mocks.MockStorageTransformer{Address: common.BytesToAddress(address)}
watcher.AddTransformers([]storage.TransformerInitializer{fakeTransformer.FakeTransformerInitializer})
assert(func(err error) {
Expect(err).To(BeNil())
expectedRow, err := shared2.FromStrings(strings.Split(line.Text, ","))
Expect(err).NotTo(HaveOccurred())
Expect(fakeTransformer.PassedRow).To(Equal(expectedRow))
}, watcher, mockTailer, []*tail.Line{line})
})
It("logs error if executing transformer fails", func() {
address := []byte{1, 2, 3}
blockHash := []byte{4, 5, 6}
blockHeight := int64(789)
storageKey := []byte{9, 8, 7}
storageValue := []byte{6, 5, 4}
mockTailer := fakes.NewMockTailer()
line := &tail.Line{
Text: fmt.Sprintf("%s,%s,%d,%s,%s", common.Bytes2Hex(address), common.Bytes2Hex(blockHash), blockHeight, common.Bytes2Hex(storageKey), common.Bytes2Hex(storageValue)),
Time: time.Time{},
Err: nil,
}
watcher := shared.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
executionError := errors.New("storage watcher failed attempting to execute transformer")
fakeTransformer := &mocks.MockStorageTransformer{Address: common.BytesToAddress(address), ExecuteErr: executionError}
watcher.AddTransformers([]storage.TransformerInitializer{fakeTransformer.FakeTransformerInitializer})
tempFile, err := ioutil.TempFile("", "log")
defer os.Remove(tempFile.Name())
Expect(err).NotTo(HaveOccurred())
logrus.SetOutput(tempFile)
assert(func(err error) {
Expect(err).NotTo(HaveOccurred())
logContent, readErr := ioutil.ReadFile(tempFile.Name())
Expect(readErr).NotTo(HaveOccurred())
Expect(string(logContent)).To(ContainSubstring(executionError.Error()))
}, watcher, mockTailer, []*tail.Line{line})
})
})
func assert(assertion func(err error), watcher shared.StorageWatcher, mockTailer *fakes.MockTailer, lines []*tail.Line) {
errs := make(chan error, 1)
done := make(chan bool, 1)
go execute(watcher, mockTailer, errs, done)
for _, line := range lines {
mockTailer.Lines <- line
}
close(mockTailer.Lines)
select {
case err := <-errs:
assertion(err)
break
case <-done:
assertion(nil)
break
}
}
func execute(watcher shared.StorageWatcher, tailer *fakes.MockTailer, errs chan error, done chan bool) {
err := watcher.Execute()
if err != nil {
errs <- err
} else {
done <- true
}
}

2
pkg/datastore/postgres/repositories/header_repository.go
View File

@ -52,7 +52,7 @@ func (repository HeaderRepository) CreateOrUpdateHeader(header core.Header) (int
func (repository HeaderRepository) GetHeader(blockNumber int64) (core.Header, error) {
var header core.Header
err := repository.database.Get(&header, `SELECT block_number, hash, raw FROM headers WHERE block_number = $1 AND eth_node_fingerprint = $2`,
err := repository.database.Get(&header, `SELECT id, block_number, hash, raw, block_timestamp FROM headers WHERE block_number = $1 AND eth_node_fingerprint = $2`,
blockNumber, repository.database.Node.ID)
return header, err
}

2
pkg/datastore/postgres/repositories/header_repository_test.go
View File

@ -188,9 +188,11 @@ var _ = Describe("Block header repository", func() {
dbHeader, err := repo.GetHeader(header.BlockNumber)
Expect(err).NotTo(HaveOccurred())
Expect(dbHeader.Id).NotTo(BeZero())
Expect(dbHeader.BlockNumber).To(Equal(header.BlockNumber))
Expect(dbHeader.Hash).To(Equal(header.Hash))
Expect(dbHeader.Raw).To(MatchJSON(header.Raw))
Expect(dbHeader.Timestamp).To(Equal(header.Timestamp))
})
It("does not return header for a different node fingerprint", func() {

16
pkg/fakes/mock_header_repository.go
View File

@ -27,8 +27,11 @@ type MockHeaderRepository struct {
createOrUpdateHeaderErr error
createOrUpdateHeaderPassedBlockNumbers []int64
createOrUpdateHeaderReturnID int64
getHeaderError error
getHeaderReturnBlockHash string
missingBlockNumbers []int64
headerExists bool
GetHeaderPassedBlockNumber int64
}
func NewMockHeaderRepository() *MockHeaderRepository {
@ -53,8 +56,9 @@ func (repository *MockHeaderRepository) CreateOrUpdateHeader(header core.Header)
return repository.createOrUpdateHeaderReturnID, repository.createOrUpdateHeaderErr
}
func (*MockHeaderRepository) GetHeader(blockNumber int64) (core.Header, error) {
return core.Header{BlockNumber: blockNumber}, nil
func (repository *MockHeaderRepository) GetHeader(blockNumber int64) (core.Header, error) {
repository.GetHeaderPassedBlockNumber = blockNumber
return core.Header{BlockNumber: blockNumber, Hash: repository.getHeaderReturnBlockHash}, repository.getHeaderError
}
func (repository *MockHeaderRepository) MissingBlockNumbers(startingBlockNumber, endingBlockNumber int64, nodeID string) ([]int64, error) {
@ -69,6 +73,14 @@ func (repository *MockHeaderRepository) SetHeaderExists(headerExists bool) {
repository.headerExists = headerExists
}
func (repository *MockHeaderRepository) SetGetHeaderError(err error) {
repository.getHeaderError = err
}
func (repository *MockHeaderRepository) SetGetHeaderReturnBlockHash(hash string) {
repository.getHeaderReturnBlockHash = hash
}
func (repository *MockHeaderRepository) AssertCreateOrUpdateHeaderCallCountAndPassedBlockNumbers(times int, blockNumbers []int64) {
Expect(repository.createOrUpdateHeaderCallCount).To(Equal(times))
Expect(repository.createOrUpdateHeaderPassedBlockNumbers).To(Equal(blockNumbers))

29
pkg/fakes/mock_tailer.go Normal file
View File

@ -0,0 +1,29 @@
package fakes
import (
"github.com/hpcloud/tail"
"gopkg.in/tomb.v1"
)
type MockTailer struct {
Lines chan *tail.Line
TailCalled bool
}
func NewMockTailer() *MockTailer {
return &MockTailer{
Lines: make(chan *tail.Line, 1),
TailCalled: false,
}
}
func (mock *MockTailer) Tail() (*tail.Tail, error) {
mock.TailCalled = true
fakeTail := &tail.Tail{
Filename: "",
Lines: mock.Lines,
Config: tail.Config{},
Tomb: tomb.Tomb{},
}
return fakeTail, nil
}

3
pkg/fs/reader.go
View File

@ -22,8 +22,7 @@ type Reader interface {
Read(path string) ([]byte, error)
}
type FsReader struct {
}
type FsReader struct{}
func (FsReader) Read(path string) ([]byte, error) {
return ioutil.ReadFile(path)

15
pkg/fs/tail.go Normal file
View File

@ -0,0 +1,15 @@
package fs
import "github.com/hpcloud/tail"
type Tailer interface {
Tail() (*tail.Tail, error)
}
type FileTailer struct {
Path string
}
func (tailer FileTailer) Tail() (*tail.Tail, error) {
return tail.TailFile(tailer.Path, tail.Config{Follow: true})
}

3
pkg/transformers/bite/converter.go
View File

@ -17,7 +17,6 @@
package bite
import (
"bytes"
"encoding/json"
"fmt"
"github.com/ethereum/go-ethereum/accounts/abi/bind"
@ -64,7 +63,7 @@ func (converter BiteConverter) ToModels(entities []interface{}) ([]interface{},
return nil, fmt.Errorf("entity of type %T, not %T", entity, BiteEntity{})
}
ilk := string(bytes.Trim(biteEntity.Ilk[:], "\x00"))
ilk := common.Bytes2Hex(biteEntity.Ilk[:])
urn := common.BytesToAddress(biteEntity.Urn[:]).String()
ink := biteEntity.Ink
art := biteEntity.Art

2
pkg/transformers/bite/converter_test.go
View File

@ -70,7 +70,7 @@ var _ = Describe("Bite Converter", func() {
emptyLog, err := json.Marshal(types.Log{})
Expect(err).NotTo(HaveOccurred())
expectedModel := bite.BiteModel{
Ilk: "",
Ilk: "0000000000000000000000000000000000000000000000000000000000000000",
Urn: "0x0000000000000000000000000000000000000000",
Ink: "",
Art: "",

2
pkg/transformers/cat_file/chop_lump/converter.go
View File

@ -40,7 +40,7 @@ func (CatFileChopLumpConverter) ToModels(ethLogs []types.Log) ([]interface{}, er
if err != nil {
return nil, err
}
ilk := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
ilk := shared.GetHexWithoutPrefix(ethLog.Topics[2].Bytes())
what := string(bytes.Trim(ethLog.Topics[3].Bytes(), "\x00"))
dataBytes := ethLog.Data[len(ethLog.Data)-constants.DataItemLength:]
data := big.NewInt(0).SetBytes(dataBytes).String()

3
pkg/transformers/cat_file/flip/converter.go
View File

@ -22,6 +22,7 @@ import (
"errors"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
)
@ -34,7 +35,7 @@ func (CatFileFlipConverter) ToModels(ethLogs []types.Log) ([]interface{}, error)
if err != nil {
return nil, err
}
ilk := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
ilk := shared.GetHexWithoutPrefix(ethLog.Topics[2].Bytes())
what := string(bytes.Trim(ethLog.Topics[3].Bytes(), "\x00"))
flipBytes := ethLog.Data[len(ethLog.Data)-constants.DataItemLength:]
flip := common.BytesToAddress(flipBytes).String()

4
pkg/transformers/drip_drip/converter.go
View File

@ -17,10 +17,10 @@
package drip_drip
import (
"bytes"
"encoding/json"
"errors"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
)
type DripDripConverter struct{}
@ -32,7 +32,7 @@ func (DripDripConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) {
if err != nil {
return nil, err
}
ilk := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
ilk := shared.GetHexWithoutPrefix(ethLog.Topics[2].Bytes())
raw, err := json.Marshal(ethLog)
if err != nil {
return nil, err

2
pkg/transformers/drip_file/ilk/converter.go
View File

@ -38,7 +38,7 @@ func (DripFileIlkConverter) ToModels(ethLogs []types.Log) ([]interface{}, error)
return nil, err
}
ilk := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
ilk := shared.GetHexWithoutPrefix(ethLog.Topics[2].Bytes())
vow := string(bytes.Trim(ethLog.Topics[3].Bytes(), "\x00"))
taxBytes := ethLog.Data[len(ethLog.Data)-constants.DataItemLength:]
tax := shared.ConvertToRay(big.NewInt(0).SetBytes(taxBytes).String())

29
pkg/transformers/factories/storage/storage_suite_test.go Normal file
View File

@ -0,0 +1,29 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package storage_test
import (
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
func TestStorage(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Storage Suite")
}

53
pkg/transformers/factories/storage/transformer.go Normal file
View File

@ -0,0 +1,53 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package storage
import (
"github.com/ethereum/go-ethereum/common"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/storage"
"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs"
"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
)
type Transformer struct {
Address common.Address
Mappings storage_diffs.Mappings
Repository storage_diffs.Repository
}
func (transformer Transformer) NewTransformer(db *postgres.DB) storage.Transformer {
transformer.Mappings.SetDB(db)
transformer.Repository.SetDB(db)
return transformer
}
func (transformer Transformer) ContractAddress() common.Address {
return transformer.Address
}
func (transformer Transformer) Execute(row shared.StorageDiffRow) error {
metadata, lookupErr := transformer.Mappings.Lookup(row.StorageKey)
if lookupErr != nil {
return lookupErr
}
value, decodeErr := shared.Decode(row, metadata)
if decodeErr != nil {
return decodeErr
}
return transformer.Repository.Create(row.BlockHeight, row.BlockHash.Hex(), metadata, value)
}

102
pkg/transformers/factories/storage/transformer_test.go Normal file
View File

@ -0,0 +1,102 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package storage_test
import (
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/transformers/factories/storage"
"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks"
)
var _ = Describe("Storage transformer", func() {
var (
mappings *mocks.MockMappings
repository *mocks.MockStorageRepository
transformer storage.Transformer
)
BeforeEach(func() {
mappings = &mocks.MockMappings{}
repository = &mocks.MockStorageRepository{}
transformer = storage.Transformer{
Address: common.Address{},
Mappings: mappings,
Repository: repository,
}
})
It("returns the contract address being watched", func() {
fakeAddress := common.HexToAddress("0x12345")
transformer.Address = fakeAddress
Expect(transformer.ContractAddress()).To(Equal(fakeAddress))
})
It("looks up metadata for storage key", func() {
transformer.Execute(shared.StorageDiffRow{})
Expect(mappings.LookupCalled).To(BeTrue())
})
It("returns error if lookup fails", func() {
mappings.LookupErr = fakes.FakeError
err := transformer.Execute(shared.StorageDiffRow{})
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("creates storage row with decoded data", func() {
fakeMetadata := shared.StorageValueMetadata{Type: shared.Address}
mappings.Metadata = fakeMetadata
rawValue := common.HexToAddress("0x12345")
fakeBlockNumber := 123
fakeBlockHash := "0x67890"
fakeRow := shared.StorageDiffRow{
Contract: common.Address{},
BlockHash: common.HexToHash(fakeBlockHash),
BlockHeight: fakeBlockNumber,
StorageKey: common.Hash{},
StorageValue: rawValue.Hash(),
}
err := transformer.Execute(fakeRow)
Expect(err).NotTo(HaveOccurred())
Expect(repository.PassedBlockNumber).To(Equal(fakeBlockNumber))
Expect(repository.PassedBlockHash).To(Equal(common.HexToHash(fakeBlockHash).Hex()))
Expect(repository.PassedMetadata).To(Equal(fakeMetadata))
Expect(repository.PassedValue.(string)).To(Equal(rawValue.Hex()))
})
It("returns error if creating row fails", func() {
rawValue := common.HexToAddress("0x12345")
fakeMetadata := shared.StorageValueMetadata{Type: shared.Address}
mappings.Metadata = fakeMetadata
repository.CreateErr = fakes.FakeError
err := transformer.Execute(shared.StorageDiffRow{StorageValue: rawValue.Hash()})
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
})

3
pkg/transformers/frob/converter.go
View File

@ -17,7 +17,6 @@
package frob
import (
"bytes"
"encoding/json"
"fmt"
log "github.com/sirupsen/logrus"
@ -68,7 +67,7 @@ func (FrobConverter) ToModels(entities []interface{}) ([]interface{}, error) {
return nil, err
}
model := FrobModel{
Ilk: string(bytes.Trim(frobEntity.Ilk[:], "\x00)")),
Ilk: common.Bytes2Hex(frobEntity.Ilk[:]),
Urn: common.BytesToAddress(frobEntity.Urn[:]).String(),
Ink: frobEntity.Ink.String(),
Art: frobEntity.Art.String(),

2
pkg/transformers/integration_tests/bite.go
View File

@ -83,7 +83,7 @@ var _ = Describe("Bite Transformer", func() {
Expect(len(dbResult)).To(Equal(1))
Expect(dbResult[0].Art).To(Equal("149846666666666655744"))
Expect(dbResult[0].IArt).To(Equal("1645356666666666655736"))
Expect(dbResult[0].Ilk).To(Equal("ETH"))
Expect(dbResult[0].Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000"))
Expect(dbResult[0].Ink).To(Equal("1000000000000000000"))
Expect(dbResult[0].NFlip).To(Equal("2"))
Expect(dbResult[0].Tab).To(Equal("149846666666666655744"))

6
pkg/transformers/integration_tests/cat_file.go
View File

@ -97,12 +97,12 @@ var _ = Describe("Cat File transformer", func() {
Expect(len(dbResult)).To(Equal(2))
sort.Sort(byLogIndexChopLump(dbResult))
Expect(dbResult[0].Ilk).To(Equal("REP"))
Expect(dbResult[0].Ilk).To(Equal("5245500000000000000000000000000000000000000000000000000000000000"))
Expect(dbResult[0].What).To(Equal("lump"))
Expect(dbResult[0].Data).To(Equal("10000.000000000000000000"))
Expect(dbResult[0].LogIndex).To(Equal(uint(3)))
Expect(dbResult[1].Ilk).To(Equal("REP"))
Expect(dbResult[1].Ilk).To(Equal("5245500000000000000000000000000000000000000000000000000000000000"))
Expect(dbResult[1].What).To(Equal("chop"))
Expect(dbResult[1].Data).To(Equal("1.000000000000000000000000000"))
Expect(dbResult[1].LogIndex).To(Equal(uint(4)))
@ -145,7 +145,7 @@ var _ = Describe("Cat File transformer", func() {
Expect(err).NotTo(HaveOccurred())
Expect(len(dbResult)).To(Equal(1))
Expect(dbResult[0].Ilk).To(Equal("ETH"))
Expect(dbResult[0].Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000"))
Expect(dbResult[0].What).To(Equal("flip"))
Expect(dbResult[0].Flip).To(Equal("0x32D496Ad866D110060866B7125981C73642cc509"))
})

2
pkg/transformers/integration_tests/drip_drip.go
View File

@ -85,6 +85,6 @@ var _ = Describe("DripDrip Transformer", func() {
Expect(len(dbResults)).To(Equal(1))
dbResult := dbResults[0]
Expect(dbResult.Ilk).To(Equal("ETH"))
Expect(dbResult.Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000"))
})
})

2
pkg/transformers/integration_tests/frob.go
View File

@ -94,7 +94,7 @@ var _ = Describe("Frob Transformer", func() {
Expect(dbResult[0].Dart).To(Equal("0"))
Expect(dbResult[0].Dink).To(Equal("10000000000000"))
Expect(dbResult[0].IArt).To(Equal("1495509999999999999992"))
Expect(dbResult[0].Ilk).To(Equal("ETH"))
Expect(dbResult[0].Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000"))
Expect(dbResult[0].Ink).To(Equal("10050100000000000"))
Expect(dbResult[0].Urn).To(Equal("0xc8E093e5f3F9B5Aa6A6b33ea45960b93C161430C"))
})

4
pkg/transformers/integration_tests/pit_file_ilk.go
View File

@ -87,7 +87,7 @@ var _ = Describe("PitFileIlk LogNoteTransformer", func() {
Expect(err).NotTo(HaveOccurred())
Expect(len(dbResult)).To(Equal(1))
Expect(dbResult[0].Ilk).To(Equal("ETH"))
Expect(dbResult[0].Ilk).To(Equal("0x4554480000000000000000000000000000000000000000000000000000000000"))
Expect(dbResult[0].What).To(Equal("spot"))
Expect(dbResult[0].Data).To(Equal("139.840000000000003410605131648"))
})
@ -119,7 +119,7 @@ var _ = Describe("PitFileIlk LogNoteTransformer", func() {
pitFileIlkLineModel = result
}
}
Expect(pitFileIlkLineModel.Ilk).To(Equal("REP"))
Expect(pitFileIlkLineModel.Ilk).To(Equal("0x5245500000000000000000000000000000000000000000000000000000000000"))
Expect(pitFileIlkLineModel.Data).To(Equal("2000000.000000000000000000"))
})
})

2
pkg/transformers/integration_tests/vat_flux.go
View File

@ -74,7 +74,7 @@ var _ = Describe("VatFlux LogNoteTransformer", func() {
Expect(err).NotTo(HaveOccurred())
Expect(len(dbResult)).To(Equal(1))
Expect(dbResult[0].Ilk).To(Equal("REP"))
Expect(dbResult[0].Ilk).To(Equal("5245500000000000000000000000000000000000000000000000000000000000"))
Expect(dbResult[0].Src).To(Equal("0xC0851F73CC8DD5c0765E71980eC7E7Fd1EF74434"))
Expect(dbResult[0].Dst).To(Equal("0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB"))
Expect(dbResult[0].Rad).To(Equal("1800000000000000000000000000000000000000000000"))

2
pkg/transformers/integration_tests/vat_fold.go
View File

@ -82,7 +82,7 @@ var _ = Describe("VatFold Transformer", func() {
Expect(len(dbResults)).To(Equal(1))
dbResult := dbResults[0]
Expect(dbResult.Ilk).To(Equal("REP"))
Expect(dbResult.Ilk).To(Equal("5245500000000000000000000000000000000000000000000000000000000000"))
Expect(dbResult.Urn).To(Equal(common.HexToAddress("0x0000000000000000000000003728e9777b2a0a611ee0f89e00e01044ce4736d1").String()))
Expect(dbResult.Rate).To(Equal("0.000000000000000000000000000"))
})

2
pkg/transformers/integration_tests/vat_grab.go
View File

@ -75,7 +75,7 @@ var _ = Describe("Vat Grab Transformer", func() {
Expect(err).NotTo(HaveOccurred())
Expect(len(dbResult)).To(Equal(1))
Expect(dbResult[0].Ilk).To(Equal("REP"))
Expect(dbResult[0].Ilk).To(Equal("5245500000000000000000000000000000000000000000000000000000000000"))
Expect(dbResult[0].Urn).To(Equal("0x6a3AE20C315E845B2E398e68EfFe39139eC6060C"))
Expect(dbResult[0].V).To(Equal("0x2F34f22a00eE4b7a8F8BBC4eAee1658774c624e0")) //cat contract address
Expect(dbResult[0].W).To(Equal("0x3728e9777B2a0a611ee0F89e00E01044ce4736d1"))

2
pkg/transformers/integration_tests/vat_init.go
View File

@ -74,6 +74,6 @@ var _ = Describe("VatInit LogNoteTransformer", func() {
Expect(len(dbResults)).To(Equal(1))
dbResult := dbResults[0]
Expect(dbResult.Ilk).To(Equal("ETH"))
Expect(dbResult.Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000"))
})
})

2
pkg/transformers/integration_tests/vat_slip.go
View File

@ -66,7 +66,7 @@ var _ = Describe("Vat slip transformer", func() {
var model vat_slip.VatSlipModel
err = db.Get(&model, `SELECT ilk, guy, rad, tx_idx FROM maker.vat_slip WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(model.Ilk).To(Equal("ETH"))
Expect(model.Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000"))
Expect(model.Guy).To(Equal("0xDA15dCE70ab462E66779f23ee14F21d993789eE3"))
Expect(model.Rad).To(Equal("100000000000000000000000000000000000000000000000"))
Expect(model.TransactionIndex).To(Equal(uint(0)))

2
pkg/transformers/integration_tests/vat_tune.go
View File

@ -75,7 +75,7 @@ var _ = Describe("VatTune LogNoteTransformer", func() {
Expect(err).NotTo(HaveOccurred())
Expect(len(dbResult)).To(Equal(1))
Expect(dbResult[0].Ilk).To(Equal("ETH"))
Expect(dbResult[0].Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000"))
Expect(dbResult[0].Urn).To(Equal("0x4F26FfBe5F04ED43630fdC30A87638d53D0b0876"))
Expect(dbResult[0].V).To(Equal("0x4F26FfBe5F04ED43630fdC30A87638d53D0b0876"))
Expect(dbResult[0].W).To(Equal("0x4F26FfBe5F04ED43630fdC30A87638d53D0b0876"))

2
pkg/transformers/pit_file/ilk/converter.go
View File

@ -37,7 +37,7 @@ func (PitFileIlkConverter) ToModels(ethLogs []types.Log) ([]interface{}, error)
if err != nil {
return nil, err
}
ilk := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
ilk := ethLog.Topics[2].Hex()
what := string(bytes.Trim(ethLog.Topics[3].Bytes(), "\x00"))
dataBytes := ethLog.Data[len(ethLog.Data)-constants.DataItemLength:]
data, err := getData(dataBytes, what)

14
pkg/transformers/shared/storage/transformer.go Normal file
View File

@ -0,0 +1,14 @@
package storage
import (
"github.com/ethereum/go-ethereum/common"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
)
type Transformer interface {
Execute(row shared.StorageDiffRow) error
ContractAddress() common.Address
}
type TransformerInitializer func(db *postgres.DB) Transformer

5
pkg/transformers/shared/utilities.go
View File

@ -17,6 +17,7 @@
package shared
import (
"github.com/ethereum/go-ethereum/common"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
"math/big"
)
@ -59,6 +60,10 @@ func GetDataBytesAtIndex(n int, logData []byte) []byte {
return []byte{}
}
func GetHexWithoutPrefix(raw []byte) string {
return common.Bytes2Hex(raw)
}
func ConvertToRay(value string) string {
return convert(ray, value, rayPrecision)
}

8
pkg/transformers/shared/utilities_test.go
View File

@ -62,4 +62,12 @@ var _ = Describe("Shared utilities", func() {
Expect(wadTwo).To(Equal("1.234567890123456690"))
})
})
Describe("getting hex without prefix", func() {
It("returns bytes as hex without 0x prefix", func() {
raw := common.HexToHash("0x4554480000000000000000000000000000000000000000000000000000000000").Bytes()
result := shared.GetHexWithoutPrefix(raw)
Expect(result).To(Equal("4554480000000000000000000000000000000000000000000000000000000000"))
})
})
})

38
pkg/transformers/storage_diffs/maker/maker_storage_repository.go Normal file
View File

@ -0,0 +1,38 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package maker
import "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
type IMakerStorageRepository interface {
GetIlks() ([]string, error)
SetDB(db *postgres.DB)
}
type MakerStorageRepository struct {
db *postgres.DB
}
func (repository *MakerStorageRepository) SetDB(db *postgres.DB) {
repository.db = db
}
func (repository MakerStorageRepository) GetIlks() ([]string, error) {
var ilks []string
err := repository.db.Select(&ilks, `SELECT DISTINCT ilk FROM maker.vat_init`)
return ilks, err
}

57
pkg/transformers/storage_diffs/maker/maker_storage_repository_test.go Normal file
View File

@ -0,0 +1,57 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package maker_test
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/maker"
"github.com/vulcanize/vulcanizedb/test_config"
)
var _ = Describe("Maker storage repository", func() {
It("fetches unique ilks from vat init events", func() {
db := test_config.NewTestDB(test_config.NewTestNode())
test_config.CleanTestDB(db)
insertVatInit("ilk1", 1, db)
insertVatInit("ilk2", 2, db)
insertVatInit("ilk2", 3, db)
repository := maker.MakerStorageRepository{}
repository.SetDB(db)
ilks, err := repository.GetIlks()
Expect(err).NotTo(HaveOccurred())
Expect(len(ilks)).To(Equal(2))
Expect(ilks).To(ConsistOf("ilk1", "ilk2"))
})
})
func insertVatInit(ilk string, blockNumber int64, db *postgres.DB) {
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(blockNumber))
Expect(err).NotTo(HaveOccurred())
_, execErr := db.Exec(
`INSERT INTO maker.vat_init (header_id, ilk, log_idx, tx_idx, raw_log)
VALUES($1, $2, $3, $4, $5)`,
headerID, ilk, 0, 0, "[]",
)
Expect(execErr).NotTo(HaveOccurred())
}

29
pkg/transformers/storage_diffs/maker/maker_suite_test.go Normal file
View File

@ -0,0 +1,29 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package maker_test
import (
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
func TestMaker(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Maker Suite")
}

146
pkg/transformers/storage_diffs/maker/pit/mappings.go Normal file
View File

@ -0,0 +1,146 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package pit
import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs"
"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/maker"
"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
"math/big"
)
const (
IlkLine = "line"
IlkSpot = "spot"
PitDrip = "drip"
PitLine = "Line"
PitLive = "live"
PitVat = "vat"
)
var (
// storage key and value metadata for "drip" on the Pit contract
DripKey = common.HexToHash(storage_diffs.IndexFive)
DripMetadata = shared.StorageValueMetadata{
Name: PitDrip,
Key: "",
Type: shared.Address,
}
IlkSpotIndex = storage_diffs.IndexOne
// storage key and value metadata for "Spot" on the Pit contract
LineKey = common.HexToHash(storage_diffs.IndexThree)
LineMetadata = shared.StorageValueMetadata{
Name: PitLine,
Key: "",
Type: shared.Uint256,
}
// storage key and value metadata for "live" on the Pit contract
LiveKey = common.HexToHash(storage_diffs.IndexTwo)
LiveMetadata = shared.StorageValueMetadata{
Name: PitLive,
Key: "",
Type: shared.Uint256,
}
// storage key and value metadata for "vat" on the Pit contract
VatKey = common.HexToHash(storage_diffs.IndexFour)
VatMetadata = shared.StorageValueMetadata{
Name: PitVat,
Key: "",
Type: shared.Address,
}
)
type PitMappings struct {
StorageRepository maker.IMakerStorageRepository
mappings map[common.Hash]shared.StorageValueMetadata
}
func (mappings *PitMappings) SetDB(db *postgres.DB) {
mappings.StorageRepository.SetDB(db)
}
func (mappings *PitMappings) Lookup(key common.Hash) (shared.StorageValueMetadata, error) {
metadata, ok := mappings.mappings[key]
if !ok {
err := mappings.loadMappings()
if err != nil {
return metadata, err
}
metadata, ok = mappings.mappings[key]
if !ok {
return metadata, shared.ErrStorageKeyNotFound{Key: key.Hex()}
}
}
return metadata, nil
}
func (mappings *PitMappings) loadMappings() error {
mappings.mappings = getStaticMappings()
ilks, err := mappings.StorageRepository.GetIlks()
if err != nil {
return err
}
for _, ilk := range ilks {
mappings.mappings[getSpotKey(ilk)] = getSpotMetadata(ilk)
mappings.mappings[getLineKey(ilk)] = getLineMetadata(ilk)
}
return nil
}
func getStaticMappings() map[common.Hash]shared.StorageValueMetadata {
mappings := make(map[common.Hash]shared.StorageValueMetadata)
mappings[DripKey] = DripMetadata
mappings[LineKey] = LineMetadata
mappings[LiveKey] = LiveMetadata
mappings[VatKey] = VatMetadata
return mappings
}
func getSpotKey(ilk string) common.Hash {
keyBytes := common.FromHex("0x" + ilk + IlkSpotIndex)
encoded := crypto.Keccak256(keyBytes)
return common.BytesToHash(encoded)
}
func getSpotMetadata(ilk string) shared.StorageValueMetadata {
return shared.StorageValueMetadata{
Name: IlkSpot,
Key: ilk,
Type: shared.Uint256,
}
}
func getLineKey(ilk string) common.Hash {
spotMappingAsInt := big.NewInt(0).SetBytes(getSpotKey(ilk).Bytes())
incrementedByOne := big.NewInt(0).Add(spotMappingAsInt, big.NewInt(1))
return common.BytesToHash(incrementedByOne.Bytes())
}
func getLineMetadata(ilk string) shared.StorageValueMetadata {
return shared.StorageValueMetadata{
Name: IlkLine,
Key: ilk,
Type: shared.Uint256,
}
}

90
pkg/transformers/storage_diffs/maker/pit/mappings_test.go Normal file
View File

@ -0,0 +1,90 @@
package pit_test
import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/maker/pit"
"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/maker/test_helpers"
"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
"math/big"
)
var _ = Describe("Pit storage mappings", func() {
Describe("looking up static keys", func() {
It("returns value metadata if key exists", func() {
storageRepository := &test_helpers.MockMakerStorageRepository{}
mappings := pit.PitMappings{StorageRepository: storageRepository}
Expect(mappings.Lookup(pit.DripKey)).To(Equal(pit.DripMetadata))
Expect(mappings.Lookup(pit.LineKey)).To(Equal(pit.LineMetadata))
Expect(mappings.Lookup(pit.LiveKey)).To(Equal(pit.LiveMetadata))
Expect(mappings.Lookup(pit.VatKey)).To(Equal(pit.VatMetadata))
})
It("returns error if key does not exist", func() {
mappings := pit.PitMappings{StorageRepository: &test_helpers.MockMakerStorageRepository{}}
_, err := mappings.Lookup(common.HexToHash(fakes.FakeHash.Hex()))
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(shared.ErrStorageKeyNotFound{Key: fakes.FakeHash.Hex()}))
})
})
Describe("looking up dynamic keys", func() {
It("refreshes mappings from repository if key not found", func() {
storageRepository := &test_helpers.MockMakerStorageRepository{}
mappings := pit.PitMappings{StorageRepository: storageRepository}
mappings.Lookup(fakes.FakeHash)
Expect(storageRepository.GetIlksCalled).To(BeTrue())
})
It("returns value metadata for spot when ilk in the DB", func() {
storageRepository := &test_helpers.MockMakerStorageRepository{}
fakeIlk := "fakeIlk"
storageRepository.SetIlks([]string{fakeIlk})
mappings := pit.PitMappings{StorageRepository: storageRepository}
ilkSpotKey := common.BytesToHash(crypto.Keccak256(common.FromHex("0x" + fakeIlk + pit.IlkSpotIndex)))
expectedMetadata := shared.StorageValueMetadata{
Name: pit.IlkSpot,
Key: fakeIlk,
Type: shared.Uint256,
}
Expect(mappings.Lookup(ilkSpotKey)).To(Equal(expectedMetadata))
})
It("returns value metadata for line when ilk in the DB", func() {
storageRepository := &test_helpers.MockMakerStorageRepository{}
fakeIlk := "fakeIlk"
storageRepository.SetIlks([]string{fakeIlk})
mappings := pit.PitMappings{StorageRepository: storageRepository}
ilkSpotKeyBytes := crypto.Keccak256(common.FromHex("0x" + fakeIlk + pit.IlkSpotIndex))
ilkSpotAsInt := big.NewInt(0).SetBytes(ilkSpotKeyBytes)
incrementedIlkSpot := big.NewInt(0).Add(ilkSpotAsInt, big.NewInt(1))
ilkLineKey := common.BytesToHash(incrementedIlkSpot.Bytes())
expectedMetadata := shared.StorageValueMetadata{
Name: pit.IlkLine,
Key: fakeIlk,
Type: shared.Uint256,
}
Expect(mappings.Lookup(ilkLineKey)).To(Equal(expectedMetadata))
})
It("returns error if key not found", func() {
storageRepository := &test_helpers.MockMakerStorageRepository{}
mappings := pit.PitMappings{StorageRepository: storageRepository}
_, err := mappings.Lookup(fakes.FakeHash)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(shared.ErrStorageKeyNotFound{Key: fakes.FakeHash.Hex()}))
})
})
})

13
pkg/transformers/storage_diffs/maker/pit/pit_suite_test.go Normal file
View File

@ -0,0 +1,13 @@
package pit_test
import (
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
func TestPit(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Pit Suite")
}

79
pkg/transformers/storage_diffs/maker/pit/repository.go Normal file
View File

@ -0,0 +1,79 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package pit
import (
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
)
type PitStorageRepository struct {
db *postgres.DB
}
func (repository *PitStorageRepository) SetDB(db *postgres.DB) {
repository.db = db
}
func (repository PitStorageRepository) Create(blockNumber int, blockHash string, metadata shared.StorageValueMetadata, value interface{}) error {
switch metadata.Name {
case IlkLine:
return repository.insertIlkLine(blockNumber, blockHash, metadata.Key, value.(string))
case IlkSpot:
return repository.insertIlkSpot(blockNumber, blockHash, metadata.Key, value.(string))
case PitDrip:
return repository.insertPitDrip(blockNumber, blockHash, value.(string))
case PitLine:
return repository.insertPitLine(blockNumber, blockHash, value.(string))
case PitLive:
return repository.insertPitLive(blockNumber, blockHash, value.(string))
case PitVat:
return repository.insertPitVat(blockNumber, blockHash, value.(string))
default:
panic("unrecognized storage metadata name")
}
}
func (repository PitStorageRepository) insertIlkLine(blockNumber int, blockHash string, ilk string, line string) error {
_, err := repository.db.Exec(`INSERT INTO maker.pit_ilk_line (block_number, block_hash, ilk, line) VALUES ($1, $2, $3, $4)`, blockNumber, blockHash, ilk, line)
return err
}
func (repository PitStorageRepository) insertIlkSpot(blockNumber int, blockHash string, ilk string, spot string) error {
_, err := repository.db.Exec(`INSERT INTO maker.pit_ilk_spot (block_number, block_hash, ilk, spot) VALUES ($1, $2, $3, $4)`, blockNumber, blockHash, ilk, spot)
return err
}
func (repository PitStorageRepository) insertPitDrip(blockNumber int, blockHash string, drip string) error {
_, err := repository.db.Exec(`INSERT INTO maker.pit_drip (block_number, block_hash, drip) VALUES ($1, $2, $3)`, blockNumber, blockHash, drip)
return err
}
func (repository PitStorageRepository) insertPitLine(blockNumber int, blockHash string, line string) error {
_, err := repository.db.Exec(`INSERT INTO maker.pit_line (block_number, block_hash, line) VALUES ($1, $2, $3)`, blockNumber, blockHash, line)
return err
}
func (repository PitStorageRepository) insertPitLive(blockNumber int, blockHash string, live string) error {
_, err := repository.db.Exec(`INSERT INTO maker.pit_live (block_number, block_hash, live) VALUES ($1, $2, $3)`, blockNumber, blockHash, live)
return err
}
func (repository PitStorageRepository) insertPitVat(blockNumber int, blockHash string, vat string) error {
_, err := repository.db.Exec(`INSERT INTO maker.pit_vat (block_number, block_hash, vat) VALUES ($1, $2, $3)`, blockNumber, blockHash, vat)
return err
}

172
pkg/transformers/storage_diffs/maker/pit/repository_test.go Normal file
View File

@ -0,0 +1,172 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package pit_test
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/maker/pit"
"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
"github.com/vulcanize/vulcanizedb/test_config"
)
var _ = Describe("Pit storage repository", func() {
var (
blockNumber int
blockHash string
db *postgres.DB
err error
repo pit.PitStorageRepository
)
BeforeEach(func() {
blockNumber = 123
blockHash = "expected_block_hash"
db = test_config.NewTestDB(test_config.NewTestNode())
test_config.CleanTestDB(db)
repo = pit.PitStorageRepository{}
repo.SetDB(db)
})
It("persists an ilk line", func() {
expectedIlk := "fake_ilk"
expectedLine := "12345"
ilkLineMetadata := shared.StorageValueMetadata{
Name: pit.IlkLine,
Key: expectedIlk,
Type: shared.Uint256,
}
err = repo.Create(blockNumber, blockHash, ilkLineMetadata, expectedLine)
Expect(err).NotTo(HaveOccurred())
type IlkLine struct {
BlockMetadata
Ilk string
Line string
}
var result IlkLine
err = db.Get(&result, `SELECT block_number, block_hash, ilk, line FROM maker.pit_ilk_line`)
Expect(err).NotTo(HaveOccurred())
Expect(result.BlockNumber).To(Equal(blockNumber))
Expect(result.BlockHash).To(Equal(blockHash))
Expect(result.Ilk).To(Equal(expectedIlk))
Expect(result.Line).To(Equal(expectedLine))
})
It("persists an ilk spot", func() {
expectedIlk := "fake_ilk"
expectedSpot := "12345"
ilkSpotMetadata := shared.StorageValueMetadata{
Name: pit.IlkSpot,
Key: expectedIlk,
Type: shared.Uint256,
}
err = repo.Create(blockNumber, blockHash, ilkSpotMetadata, expectedSpot)
Expect(err).NotTo(HaveOccurred())
type IlkSpot struct {
BlockMetadata
Ilk string
Spot string
}
var result IlkSpot
err = db.Get(&result, `SELECT block_number, block_hash, ilk, spot FROM maker.pit_ilk_spot`)
Expect(err).NotTo(HaveOccurred())
Expect(result.BlockNumber).To(Equal(blockNumber))
Expect(result.BlockHash).To(Equal(blockHash))
Expect(result.Ilk).To(Equal(expectedIlk))
Expect(result.Spot).To(Equal(expectedSpot))
})
It("persists a pit drip", func() {
expectedDrip := "0x0123456789abcdef0123"
err = repo.Create(blockNumber, blockHash, pit.DripMetadata, expectedDrip)
Expect(err).NotTo(HaveOccurred())
type PitDrip struct {
BlockMetadata
Drip string
}
var result PitDrip
err = db.Get(&result, `SELECT block_number, block_hash, drip FROM maker.pit_drip`)
Expect(err).NotTo(HaveOccurred())
Expect(result.BlockNumber).To(Equal(blockNumber))
Expect(result.BlockHash).To(Equal(blockHash))
Expect(result.Drip).To(Equal(expectedDrip))
})
It("persists a pit line", func() {
expectedLine := "12345"
err = repo.Create(blockNumber, blockHash, pit.LineMetadata, expectedLine)
Expect(err).NotTo(HaveOccurred())
type PitLine struct {
BlockMetadata
Line string
}
var result PitLine
err = db.Get(&result, `SELECT block_number, block_hash, line FROM maker.pit_line`)
Expect(err).NotTo(HaveOccurred())
Expect(result.BlockNumber).To(Equal(blockNumber))
Expect(result.BlockHash).To(Equal(blockHash))
Expect(result.Line).To(Equal(expectedLine))
})
It("persists a pit live", func() {
expectedLive := "12345"
err = repo.Create(blockNumber, blockHash, pit.LiveMetadata, expectedLive)
Expect(err).NotTo(HaveOccurred())
type PitLive struct {
BlockMetadata
Live string
}
var result PitLive
err = db.Get(&result, `SELECT block_number, block_hash, live FROM maker.pit_live`)
Expect(err).NotTo(HaveOccurred())
Expect(result.BlockNumber).To(Equal(blockNumber))
Expect(result.BlockHash).To(Equal(blockHash))
Expect(result.Live).To(Equal(expectedLive))
})
It("persists a pit vat", func() {
expectedVat := "0x0123456789abcdef0123"
err = repo.Create(blockNumber, blockHash, pit.VatMetadata, expectedVat)
Expect(err).NotTo(HaveOccurred())
type PitVat struct {
BlockMetadata
Vat string
}
var result PitVat
err = db.Get(&result, `SELECT block_number, block_hash, vat FROM maker.pit_vat`)
Expect(err).NotTo(HaveOccurred())
Expect(result.BlockNumber).To(Equal(blockNumber))
Expect(result.BlockHash).To(Equal(blockHash))
Expect(result.Vat).To(Equal(expectedVat))
})
})
type BlockMetadata struct {
BlockNumber int `db:"block_number"`
BlockHash string `db:"block_hash"`
}

19
pkg/transformers/storage_diffs/maker/test_helpers/maker_storage_repository.go Normal file
View File

@ -0,0 +1,19 @@
package test_helpers
import "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
type MockMakerStorageRepository struct {
GetIlksCalled bool
ilks []string
}
func (repository *MockMakerStorageRepository) GetIlks() ([]string, error) {
repository.GetIlksCalled = true
return repository.ilks, nil
}
func (repository *MockMakerStorageRepository) SetDB(db *postgres.DB) {}
func (repository *MockMakerStorageRepository) SetIlks(ilks []string) {
repository.ilks = ilks
}

39
pkg/transformers/storage_diffs/mappings.go Normal file
View File

@ -0,0 +1,39 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package storage_diffs
import (
"github.com/ethereum/go-ethereum/common"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
)
type Mappings interface {
Lookup(key common.Hash) (shared.StorageValueMetadata, error)
SetDB(db *postgres.DB)
}
const (
IndexZero = "0000000000000000000000000000000000000000000000000000000000000000"
IndexOne = "0000000000000000000000000000000000000000000000000000000000000001"
IndexTwo = "0000000000000000000000000000000000000000000000000000000000000002"
IndexThree = "0000000000000000000000000000000000000000000000000000000000000003"
IndexFour = "0000000000000000000000000000000000000000000000000000000000000004"
IndexFive = "0000000000000000000000000000000000000000000000000000000000000005"
IndexSix = "0000000000000000000000000000000000000000000000000000000000000006"
IndexSeven = "0000000000000000000000000000000000000000000000000000000000000007"
)

27
pkg/transformers/storage_diffs/repository.go Normal file
View File

@ -0,0 +1,27 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package storage_diffs
import (
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
)
type Repository interface {
Create(blockNumber int, blockHash string, metadata shared.StorageValueMetadata, value interface{}) error
SetDB(db *postgres.DB)
}

42
pkg/transformers/storage_diffs/shared/decoder.go Normal file
View File

@ -0,0 +1,42 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package shared
import (
"github.com/ethereum/go-ethereum/common"
"math/big"
)
func Decode(row StorageDiffRow, metadata StorageValueMetadata) (interface{}, error) {
switch metadata.Type {
case Uint256:
return decodeUint256(row.StorageValue.Bytes()), nil
case Address:
return decodeAddress(row.StorageValue.Bytes()), nil
default:
return nil, ErrTypeNotFound{}
}
}
func decodeUint256(raw []byte) string {
n := big.NewInt(0).SetBytes(raw)
return n.String()
}
func decodeAddress(raw []byte) string {
return common.BytesToAddress(raw).Hex()
}

58
pkg/transformers/storage_diffs/shared/decoder_test.go Normal file
View File

@ -0,0 +1,58 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package shared_test
import (
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
"math/big"
)
var _ = Describe("Storage decoder", func() {
It("decodes uint256", func() {
fakeInt := common.HexToHash("0000000000000000000000000000000000000000000000000000000000000539")
row := shared.StorageDiffRow{StorageValue: fakeInt}
metadata := shared.StorageValueMetadata{Type: shared.Uint256}
result, err := shared.Decode(row, metadata)
Expect(err).NotTo(HaveOccurred())
Expect(result).To(Equal(big.NewInt(0).SetBytes(fakeInt.Bytes()).String()))
})
It("decodes address", func() {
fakeAddress := common.HexToAddress("0x12345")
row := shared.StorageDiffRow{StorageValue: fakeAddress.Hash()}
metadata := shared.StorageValueMetadata{Type: shared.Address}
result, err := shared.Decode(row, metadata)
Expect(err).NotTo(HaveOccurred())
Expect(result).To(Equal(fakeAddress.Hex()))
})
It("returns error if attempting to decode unknown type", func() {
metadata := shared.StorageValueMetadata{Type: 100}
_, err := shared.Decode(shared.StorageDiffRow{}, metadata)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(shared.ErrTypeNotFound{}))
})
})

61
pkg/transformers/storage_diffs/shared/errors.go Normal file
View File

@ -0,0 +1,61 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package shared
import "fmt"
type ErrContractNotFound struct {
Contract string
}
func (e ErrContractNotFound) Error() string {
return fmt.Sprintf("transformer not found for contract: %s", e.Contract)
}
type ErrHeaderMismatch struct {
BlockHeight int
DbHash string
DiffHash string
}
func (e ErrHeaderMismatch) Error() string {
return fmt.Sprintf("header hash in row does not match db at height %d - row: %s, db: %s", e.BlockHeight, e.DbHash, e.DiffHash)
}
type ErrRowMalformed struct {
Length int
}
func (e ErrRowMalformed) Error() string {
return fmt.Sprintf("storage row malformed: length %d, expected %d", e.Length, ExpectedRowLength)
}
type ErrStorageKeyNotFound struct {
Key string
}
func (e ErrStorageKeyNotFound) Error() string {
return fmt.Sprintf("unknown storage key: %s", e.Key)
}
type ErrTypeNotFound struct {
Type int
}
func (e ErrTypeNotFound) Error() string {
return fmt.Sprintf("no decoder for type: %d", e.Type)
}

49
pkg/transformers/storage_diffs/shared/row.go Normal file
View File

@ -0,0 +1,49 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package shared
import (
"github.com/ethereum/go-ethereum/common"
"strconv"
)
const ExpectedRowLength = 5
type StorageDiffRow struct {
Contract common.Address
BlockHash common.Hash
BlockHeight int
StorageKey common.Hash
StorageValue common.Hash
}
func FromStrings(csvRow []string) (StorageDiffRow, error) {
if len(csvRow) != ExpectedRowLength {
return StorageDiffRow{}, ErrRowMalformed{Length: len(csvRow)}
}
height, err := strconv.Atoi(csvRow[2])
if err != nil {
return StorageDiffRow{}, err
}
return StorageDiffRow{
Contract: common.HexToAddress(csvRow[0]),
BlockHash: common.HexToHash(csvRow[1]),
BlockHeight: height,
StorageKey: common.HexToHash(csvRow[3]),
StorageValue: common.HexToHash(csvRow[4]),
}, nil
}

57
pkg/transformers/storage_diffs/shared/row_test.go Normal file
View File

@ -0,0 +1,57 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package shared_test
import (
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
)
var _ = Describe("Storage row parsing", func() {
It("converts an array of strings to a row struct", func() {
contract := "0x123"
blockHash := "0x456"
blockHeight := "789"
storageKey := "0x987"
storageValue := "0x654"
data := []string{contract, blockHash, blockHeight, storageKey, storageValue}
result, err := shared.FromStrings(data)
Expect(err).NotTo(HaveOccurred())
Expect(result.Contract).To(Equal(common.HexToAddress(contract)))
Expect(result.BlockHash).To(Equal(common.HexToHash(blockHash)))
Expect(result.BlockHeight).To(Equal(789))
Expect(result.StorageKey).To(Equal(common.HexToHash(storageKey)))
Expect(result.StorageValue).To(Equal(common.HexToHash(storageValue)))
})
It("returns an error if row is missing data", func() {
_, err := shared.FromStrings([]string{"0x123"})
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(shared.ErrRowMalformed{Length: 1}))
})
It("returns error if block height malformed", func() {
_, err := shared.FromStrings([]string{"", "", "", "", ""})
Expect(err).To(HaveOccurred())
})
})

29
pkg/transformers/storage_diffs/shared/shared_suite_test.go Normal file
View File

@ -0,0 +1,29 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package shared_test
import (
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
func TestShared(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Shared Suite")
}

31
pkg/transformers/storage_diffs/shared/value.go Normal file
View File

@ -0,0 +1,31 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package shared
type ValueType int
const (
Uint256 ValueType = iota
Bytes32
Address
)
type StorageValueMetadata struct {
Name string
Key string
Type ValueType
}

17
pkg/transformers/storage_transformers.go Normal file
View File

@ -0,0 +1,17 @@
package transformers
import (
"github.com/ethereum/go-ethereum/common"
"github.com/vulcanize/vulcanizedb/pkg/transformers/factories/storage"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/maker"
"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/maker/pit"
)
func GetPitStorageTransformer() storage.Transformer {
return storage.Transformer{
Address: common.HexToAddress(constants.PitContractAddress()),
Mappings: &pit.PitMappings{StorageRepository: &maker.MakerStorageRepository{}},
Repository: &pit.PitStorageRepository{},
}
}

2
pkg/transformers/test_data/bite.go
View File

@ -41,7 +41,7 @@ var (
biteRawJson, _ = json.Marshal(EthBiteLog)
biteIlk = [32]byte{69, 84, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
biteLad = [32]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 180, 20, 126, 218, 128, 254, 199, 18, 42, 225, 109, 162, 71, 156, 189, 127, 251}
biteIlkString = "ETH"
biteIlkString = "4554480000000000000000000000000000000000000000000000000000000000"
biteLadString = "0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB"
)

6
pkg/transformers/test_data/cat_file.go
View File

@ -45,7 +45,7 @@ var EthCatFileChopLog = types.Log{
}
var rawCatFileChopLog, _ = json.Marshal(EthCatFileChopLog)
var CatFileChopModel = chop_lump.CatFileChopLumpModel{
Ilk: "fake ilk",
Ilk: "66616b6520696c6b000000000000000000000000000000000000000000000000",
What: "chop",
Data: "123.456789012345680589533003513",
TransactionIndex: EthCatFileChopLog.TxIndex,
@ -71,7 +71,7 @@ var EthCatFileLumpLog = types.Log{
}
var rawCatFileLumpLog, _ = json.Marshal(EthCatFileLumpLog)
var CatFileLumpModel = chop_lump.CatFileChopLumpModel{
Ilk: "fake ilk",
Ilk: "66616b6520696c6b000000000000000000000000000000000000000000000000",
What: "lump",
Data: "12345.678901234567092615",
TransactionIndex: EthCatFileLumpLog.TxIndex,
@ -98,7 +98,7 @@ var EthCatFileFlipLog = types.Log{
var rawCatFileFlipLog, _ = json.Marshal(EthCatFileFlipLog)
var CatFileFlipModel = flip.CatFileFlipModel{
Ilk: "fake ilk",
Ilk: "66616b6520696c6b000000000000000000000000000000000000000000000000",
What: "flip",
Flip: "0x07Fa9eF6609cA7921112231F8f195138ebbA2977",
TransactionIndex: EthCatFileFlipLog.TxIndex,

2
pkg/transformers/test_data/drip_drip.go
View File

@ -44,7 +44,7 @@ var EthDripDripLog = types.Log{
var rawDripDripLog, _ = json.Marshal(EthDripDripLog)
var DripDripModel = drip_drip.DripDripModel{
Ilk: "fake ilk",
Ilk: "66616b6520696c6b000000000000000000000000000000000000000000000000",
LogIndex: EthDripDripLog.Index,
TransactionIndex: EthDripDripLog.TxIndex,
Raw: rawDripDripLog,

2
pkg/transformers/test_data/drip_file.go
View File

@ -47,7 +47,7 @@ var EthDripFileIlkLog = types.Log{
var rawDripFileIlkLog, _ = json.Marshal(EthDripFileIlkLog)
var DripFileIlkModel = ilk2.DripFileIlkModel{
Ilk: "fake ilk",
Ilk: "66616b6520696c6b000000000000000000000000000000000000000000000000",
Vow: "fake vow",
Tax: "12300.000000000000000000000000000",
LogIndex: EthDripFileIlkLog.Index,

2
pkg/transformers/test_data/frob.go
View File

@ -41,7 +41,7 @@ var (
frobLad = [32]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 217, 34, 137, 65, 83, 190, 158, 239, 123, 114, 24, 220, 86, 93, 29, 12, 226, 160, 146}
ink = big.NewInt(15)
ilk = [32]byte{102, 97, 107, 101, 32, 105, 108, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
frobIlkString = "fake ilk"
frobIlkString = "66616b6520696c6b000000000000000000000000000000000000000000000000"
frobUrnString = "0x64d922894153BE9EEf7b7218dc565d1D0Ce2a092"
)

22
pkg/transformers/test_data/mocks/mappings.go Normal file
View File

@ -0,0 +1,22 @@
package mocks
import (
"github.com/ethereum/go-ethereum/common"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
)
type MockMappings struct {
Metadata shared.StorageValueMetadata
LookupCalled bool
LookupErr error
}
func (mappings *MockMappings) Lookup(key common.Hash) (shared.StorageValueMetadata, error) {
mappings.LookupCalled = true
return mappings.Metadata, mappings.LookupErr
}
func (*MockMappings) SetDB(db *postgres.DB) {
panic("implement me")
}

26
pkg/transformers/test_data/mocks/storage_repository.go Normal file
View File

@ -0,0 +1,26 @@
package mocks
import (
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
)
type MockStorageRepository struct {
CreateErr error
PassedBlockNumber int
PassedBlockHash string
PassedMetadata shared.StorageValueMetadata
PassedValue interface{}
}
func (repository *MockStorageRepository) Create(blockNumber int, blockHash string, metadata shared.StorageValueMetadata, value interface{}) error {
repository.PassedBlockNumber = blockNumber
repository.PassedBlockHash = blockHash
repository.PassedMetadata = metadata
repository.PassedValue = value
return repository.CreateErr
}
func (*MockStorageRepository) SetDB(db *postgres.DB) {
panic("implement me")
}

27
pkg/transformers/test_data/mocks/storage_transformer.go Normal file
View File

@ -0,0 +1,27 @@
package mocks
import (
"github.com/ethereum/go-ethereum/common"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/storage"
"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
)
type MockStorageTransformer struct {
Address common.Address
ExecuteErr error
PassedRow shared.StorageDiffRow
}
func (transformer *MockStorageTransformer) Execute(row shared.StorageDiffRow) error {
transformer.PassedRow = row
return transformer.ExecuteErr
}
func (transformer *MockStorageTransformer) ContractAddress() common.Address {
return transformer.Address
}
func (transformer *MockStorageTransformer) FakeTransformerInitializer(db *postgres.DB) storage.Transformer {
return transformer
}

4
pkg/transformers/test_data/pit_file.go
View File

@ -75,7 +75,7 @@ var EthPitFileIlkLineLog = types.Log{
var rawPitFileIlkLineLog, _ = json.Marshal(EthPitFileIlkLineLog)
var PitFileIlkLineModel = ilk2.PitFileIlkModel{
Ilk: "fake ilk",
Ilk: "0x66616b6520696c6b000000000000000000000000000000000000000000000000",
What: "line",
Data: "0.000001000000000000",
LogIndex: EthPitFileIlkLineLog.Index,
@ -102,7 +102,7 @@ var EthPitFileIlkSpotLog = types.Log{
var rawPitFileIlkSpotLog, _ = json.Marshal(EthPitFileIlkSpotLog)
var PitFileIlkSpotModel = ilk2.PitFileIlkModel{
Ilk: "fake ilk",
Ilk: "0x66616b6520696c6b000000000000000000000000000000000000000000000000",
What: "spot",
Data: "0.000000000000001000000000000",
LogIndex: EthPitFileIlkSpotLog.Index,

2
pkg/transformers/test_data/vat_flux.go
View File

@ -44,7 +44,7 @@ var VatFluxLog = types.Log{
var rawFluxLog, _ = json.Marshal(VatFluxLog)
var VatFluxModel = vat_flux.VatFluxModel{
Ilk: "REP",
Ilk: "5245500000000000000000000000000000000000000000000000000000000000",
Src: "0x7FA9EF6609Ca7921112231f8f195138ebba29770",
Dst: "0x93086347c52a8878af71bB818509d484c6a2e1bF",
Rad: "123",

2
pkg/transformers/test_data/vat_fold.go
View File

@ -46,7 +46,7 @@ var EthVatFoldLog = types.Log{
var rawVatFoldLog, _ = json.Marshal(EthVatFoldLog)
var VatFoldModel = vat_fold.VatFoldModel{
Ilk: "REP",
Ilk: "5245500000000000000000000000000000000000000000000000000000000000",
Urn: "0x3728e9777B2a0a611ee0F89e00E01044ce4736d1",
Rate: "0.000000000000000000000000002",
LogIndex: EthVatFoldLog.Index,

2
pkg/transformers/test_data/vat_grab.go
View File

@ -28,7 +28,7 @@ var EthVatGrabLog = types.Log{
var rawVatGrabLog, _ = json.Marshal(EthVatGrabLog)
var VatGrabModel = vat_grab.VatGrabModel{
Ilk: "REP",
Ilk: "5245500000000000000000000000000000000000000000000000000000000000",
Urn: "0x6a3AE20C315E845B2E398e68EfFe39139eC6060C",
V: "0x2F34f22a00eE4b7a8F8BBC4eAee1658774c624e0",
W: "0x3728e9777B2a0a611ee0F89e00E01044ce4736d1",

2
pkg/transformers/test_data/vat_init.go
View File

@ -46,7 +46,7 @@ var EthVatInitLog = types.Log{
var rawVatInitLog, _ = json.Marshal(EthVatInitLog)
var VatInitModel = vat_init.VatInitModel{
Ilk: "fake ilk",
Ilk: "66616b6520696c6b000000000000000000000000000000000000000000000000",
LogIndex: EthVatInitLog.Index,
TransactionIndex: EthVatInitLog.TxIndex,
Raw: rawVatInitLog,

2
pkg/transformers/test_data/vat_slip.go
View File

@ -46,7 +46,7 @@ var EthVatSlipLog = types.Log{
var rawVatSlipLog, _ = json.Marshal(EthVatSlipLog)
var VatSlipModel = vat_slip.VatSlipModel{
Ilk: "fake ilk",
Ilk: "66616b6520696c6b000000000000000000000000000000000000000000000000",
Guy: common.HexToAddress("0x7d7bEe5fCfD8028cf7b00876C5b1421c800561A6").String(),
Rad: "987654321",
TransactionIndex: EthVatSlipLog.TxIndex,

2
pkg/transformers/test_data/vat_toll.go
View File

@ -29,7 +29,7 @@ var EthVatTollLog = types.Log{
var rawVatTollLog, _ = json.Marshal(EthVatTollLog)
var VatTollModel = vat_toll.VatTollModel{
Ilk: "fake ilk",
Ilk: "66616b6520696c6b000000000000000000000000000000000000000000000000",
Urn: "0xA3E37186E017747DbA34042e83e3F76Ad3CcE9b0",
Take: big.NewInt(123456789).String(),
TransactionIndex: EthVatTollLog.TxIndex,

2
pkg/transformers/test_data/vat_tune.go
View File

@ -34,7 +34,7 @@ var dartString = "11579208923731619542357098500868790785326998466564056403945558
var vatTuneDart, _ = new(big.Int).SetString(dartString, 10)
var urnAddress = "0x4F26FfBe5F04ED43630fdC30A87638d53D0b0876"
var VatTuneModel = vat_tune.VatTuneModel{
Ilk: "ETH",
Ilk: "4554480000000000000000000000000000000000000000000000000000000000",
Urn: urnAddress,
V: urnAddress,
W: urnAddress,

3
pkg/transformers/vat_flux/converter.go
View File

@ -17,7 +17,6 @@
package vat_flux
import (
"bytes"
"encoding/json"
"errors"
"github.com/ethereum/go-ethereum/common"
@ -36,7 +35,7 @@ func (VatFluxConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) {
return nil, err
}
ilk := string(bytes.Trim(ethLog.Topics[1].Bytes(), "\x00"))
ilk := shared.GetHexWithoutPrefix(ethLog.Topics[1].Bytes())
src := common.BytesToAddress(ethLog.Topics[2].Bytes())
dst := common.BytesToAddress(ethLog.Topics[3].Bytes())
radBytes := shared.GetDataBytesAtIndex(-1, ethLog.Data)

3
pkg/transformers/vat_fold/converter.go
View File

@ -17,7 +17,6 @@
package vat_fold
import (
"bytes"
"encoding/json"
"errors"
"github.com/ethereum/go-ethereum/common"
@ -35,7 +34,7 @@ func (VatFoldConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) {
return nil, err
}
ilk := string(bytes.Trim(ethLog.Topics[1].Bytes(), "\x00"))
ilk := shared.GetHexWithoutPrefix(ethLog.Topics[1].Bytes())
urn := common.BytesToAddress(ethLog.Topics[2].Bytes()).String()
rate := shared.ConvertToRay(ethLog.Topics[3].Big().String())
raw, err := json.Marshal(ethLog)

3
pkg/transformers/vat_grab/converter.go
View File

@ -1,7 +1,6 @@
package vat_grab
import (
"bytes"
"encoding/json"
"errors"
"github.com/ethereum/go-ethereum/common"
@ -20,7 +19,7 @@ func (VatGrabConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) {
if err != nil {
return nil, err
}
ilk := string(bytes.Trim(ethLog.Topics[1].Bytes(), "\x00"))
ilk := shared.GetHexWithoutPrefix(ethLog.Topics[1].Bytes())
urn := common.BytesToAddress(ethLog.Topics[2].Bytes())
v := common.BytesToAddress(ethLog.Topics[3].Bytes())
wBytes := shared.GetDataBytesAtIndex(-3, ethLog.Data)

4
pkg/transformers/vat_init/converter.go
View File

@ -17,10 +17,10 @@
package vat_init
import (
"bytes"
"encoding/json"
"errors"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
)
type VatInitConverter struct{}
@ -32,7 +32,7 @@ func (VatInitConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) {
if err != nil {
return nil, err
}
ilk := string(bytes.Trim(ethLog.Topics[1].Bytes(), "\x00"))
ilk := shared.GetHexWithoutPrefix(ethLog.Topics[1].Bytes())
raw, err := json.Marshal(ethLog)
if err != nil {
return nil, err

4
pkg/transformers/vat_slip/converter.go
View File

@ -17,11 +17,11 @@
package vat_slip
import (
"bytes"
"encoding/json"
"errors"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
)
type VatSlipConverter struct{}
@ -33,7 +33,7 @@ func (VatSlipConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) {
if err != nil {
return nil, err
}
ilk := string(bytes.Trim(ethLog.Topics[1].Bytes(), "\x00"))
ilk := shared.GetHexWithoutPrefix(ethLog.Topics[1].Bytes())
guy := common.BytesToAddress(ethLog.Topics[2].Bytes())
rad := ethLog.Topics[3].Big()

4
pkg/transformers/vat_toll/converter.go
View File

@ -1,9 +1,9 @@
package vat_toll
import (
"bytes"
"encoding/json"
"errors"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
@ -18,7 +18,7 @@ func (VatTollConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) {
if err != nil {
return nil, err
}
ilk := string(bytes.Trim(ethLog.Topics[1].Bytes(), "\x00"))
ilk := shared.GetHexWithoutPrefix(ethLog.Topics[1].Bytes())
urn := common.BytesToAddress(ethLog.Topics[2].Bytes()[:common.AddressLength])
take := ethLog.Topics[3].Big()

3
pkg/transformers/vat_tune/converter.go
View File

@ -1,7 +1,6 @@
package vat_tune
import (
"bytes"
"encoding/json"
"errors"
"math/big"
@ -22,7 +21,7 @@ func (VatTuneConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) {
if err != nil {
return nil, err
}
ilk := string(bytes.Trim(ethLog.Topics[1].Bytes(), "\x00"))
ilk := shared.GetHexWithoutPrefix(ethLog.Topics[1].Bytes())
urn := common.BytesToAddress(ethLog.Topics[2].Bytes())
v := common.BytesToAddress(ethLog.Topics[3].Bytes())
wBytes := shared.GetDataBytesAtIndex(-3, ethLog.Data)