cerc-io/ipld-eth-server
18
0
Fork 1
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases 37 Wiki Activity

Add Vat slip transformer

Browse Source
This commit is contained in:
Rob Mulholand 2018-10-15 15:04:17 -05:00
parent 3baca64903
commit 130d11d0aa
28 changed files with 973 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cmd/continuousLogSync.go
View File

@ -109,6 +109,7 @@ func buildTransformerInitializerMap() map[string]shared2.TransformerInitializer
transformerInitializerMap["vatMove"] = transformers.VatMoveTransformerInitializer
transformerInitializerMap["vatHeal"] = transformers.VatHealTransformerInitializer
transformerInitializerMap["vatFold"] = transformers.VatFoldTransformerInitializer
transformerInitializerMap["vatSlip"] = transformers.VatSlipTransformerInitializer
transformerInitializerMap["vatToll"] = transformers.VatTollTransformerInitializer
transformerInitializerMap["vatTune"] = transformers.VatTuneTransformerInitializer
transformerInitializerMap["vatFlux"] = transformers.VatFluxTransformerInitializer

2
cmd/getSignatures.go
View File

@ -53,7 +53,9 @@ func init() {
signatures["VatHealSignature"] = shared.VatHealSignature
signatures["VatGrabSignature"] = shared.VatGrabSignature
signatures["VatInitSignature"] = shared.VatInitSignature
signatures["VatFluxSignature"] = shared.VatFluxSignature
signatures["VatFoldSignature"] = shared.VatFoldSignature
signatures["VatSlipSignature"] = shared.VatSlipSignature
signatures["VatTollSignature"] = shared.VatTollSignature
signatures["VatTuneSignature"] = shared.VatTuneSignature

3
db/migrations/1539638109_create_vat_slip.down.sql Normal file
View File

@ -0,0 +1,3 @@
DROP TABLE maker.vat_slip;
ALTER TABLE public.checked_headers
DROP COLUMN vat_slip_checked;

13
db/migrations/1539638109_create_vat_slip.up.sql Normal file
View File

@ -0,0 +1,13 @@
CREATE TABLE maker.vat_slip (
id SERIAL PRIMARY KEY,
header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE,
ilk TEXT,
guy TEXT,
rad NUMERIC,
tx_idx INTEGER NOT NULL,
raw_log JSONB,
UNIQUE (header_id, tx_idx)
);
ALTER TABLE public.checked_headers
ADD COLUMN vat_slip_checked BOOLEAN NOT NULL DEFAULT FALSE;

69
db/schema.sql
View File

@ -909,6 +909,41 @@ CREATE SEQUENCE maker.vat_move_id_seq
ALTER SEQUENCE maker.vat_move_id_seq OWNED BY maker.vat_move.id;
--
-- Name: vat_slip; Type: TABLE; Schema: maker; Owner: -
--
CREATE TABLE maker.vat_slip (
id integer NOT NULL,
header_id integer NOT NULL,
ilk text,
guy text,
rad numeric,
tx_idx integer NOT NULL,
raw_log jsonb
);
--
-- Name: vat_slip_id_seq; Type: SEQUENCE; Schema: maker; Owner: -
--
CREATE SEQUENCE maker.vat_slip_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: vat_slip_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: -
--
ALTER SEQUENCE maker.vat_slip_id_seq OWNED BY maker.vat_slip.id;
--
-- Name: vat_toll; Type: TABLE; Schema: maker; Owner: -
--
@ -1089,7 +1124,8 @@ CREATE TABLE public.checked_headers (
vat_toll_checked boolean DEFAULT false NOT NULL,
vat_tune_checked boolean DEFAULT false NOT NULL,
vat_grab_checked boolean DEFAULT false NOT NULL,
vat_flux_checked boolean DEFAULT false NOT NULL
vat_flux_checked boolean DEFAULT false NOT NULL,
vat_slip_checked boolean DEFAULT false NOT NULL
);
@ -1579,6 +1615,13 @@ ALTER TABLE ONLY maker.vat_init ALTER COLUMN id SET DEFAULT nextval('maker.vat_i
ALTER TABLE ONLY maker.vat_move ALTER COLUMN id SET DEFAULT nextval('maker.vat_move_id_seq'::regclass);
--
-- Name: vat_slip id; Type: DEFAULT; Schema: maker; Owner: -
--
ALTER TABLE ONLY maker.vat_slip ALTER COLUMN id SET DEFAULT nextval('maker.vat_slip_id_seq'::regclass);
--
-- Name: vat_toll id; Type: DEFAULT; Schema: maker; Owner: -
--
@ -2047,6 +2090,22 @@ ALTER TABLE ONLY maker.vat_move
ADD CONSTRAINT vat_move_pkey PRIMARY KEY (id);
--
-- Name: vat_slip vat_slip_header_id_tx_idx_key; Type: CONSTRAINT; Schema: maker; Owner: -
--
ALTER TABLE ONLY maker.vat_slip
ADD CONSTRAINT vat_slip_header_id_tx_idx_key UNIQUE (header_id, tx_idx);
--
-- Name: vat_slip vat_slip_pkey; Type: CONSTRAINT; Schema: maker; Owner: -
--
ALTER TABLE ONLY maker.vat_slip
ADD CONSTRAINT vat_slip_pkey PRIMARY KEY (id);
--
-- Name: vat_toll vat_toll_header_id_tx_idx_key; Type: CONSTRAINT; Schema: maker; Owner: -
--
@ -2425,6 +2484,14 @@ ALTER TABLE ONLY maker.vat_move
ADD CONSTRAINT vat_move_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE;
--
-- Name: vat_slip vat_slip_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: -
--
ALTER TABLE ONLY maker.vat_slip
ADD CONSTRAINT vat_slip_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE;
--
-- Name: vat_toll vat_toll_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: -
--

1
pkg/transformers/integration_tests/frob.go
View File

@ -34,7 +34,6 @@ var _ = Describe("Frob Transformer", func() {
config.StartingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber
ipc := "https://kovan.infura.io/J5Vd2fRtGsw0zZ0Ov3BL"
rpcClient, ethClient, err := getClients(ipc)
Expect(err).NotTo(HaveOccurred())
blockchain, err := getBlockChain(rpcClient, ethClient)

22
pkg/transformers/integration_tests/helpers.go
View File

@ -15,19 +15,16 @@
package integration_tests
import (
"context"
"encoding/json"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/ethclient"
"github.com/ethereum/go-ethereum/rpc"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/geth"
"github.com/vulcanize/vulcanizedb/pkg/geth/client"
rpc2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
"github.com/vulcanize/vulcanizedb/pkg/geth/node"
"math/big"
)
func getClients(ipc string) (client.RpcClient, *ethclient.Client, error) {
@ -46,22 +43,7 @@ func getBlockChain(rpcClient client.RpcClient, ethClient *ethclient.Client) (cor
}
func persistHeader(rpcClient client.RpcClient, db *postgres.DB, blockNumber int64) error {
var poaHeader core.POAHeader
blockNumberArg := hexutil.EncodeBig(big.NewInt(int64(blockNumber)))
err := rpcClient.CallContext(context.Background(), &poaHeader, "eth_getBlockByNumber", blockNumberArg, false)
if err != nil {
return err
}
rawHeader, err := json.Marshal(poaHeader)
if err != nil {
return err
}
headerRepository := repositories.NewHeaderRepository(db)
_, err = headerRepository.CreateOrUpdateHeader(core.Header{
BlockNumber: poaHeader.Number.ToInt().Int64(),
Hash: poaHeader.Hash.String(),
Raw: rawHeader,
Timestamp: poaHeader.Time.ToInt().String(),
})
_, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(blockNumber))
return err
}

17
pkg/transformers/integration_tests/integration_tests_suite_test.go
View File

@ -1,13 +1,30 @@
package integration_tests
import (
"log"
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/spf13/viper"
"io/ioutil"
)
var ipc string
func TestIntegrationTests(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "IntegrationTests Suite")
}
var _ = BeforeSuite(func() {
testConfig := viper.New()
testConfig.SetConfigName("staging")
testConfig.AddConfigPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/environments/")
err := testConfig.ReadInConfig()
ipc = testConfig.GetString("client.ipcPath")
if err != nil {
log.Fatal(err)
}
log.SetOutput(ioutil.Discard)
})

1
pkg/transformers/integration_tests/price_feeds.go
View File

@ -33,7 +33,6 @@ var _ = Describe("Price feeds transformer", func() {
)
BeforeEach(func() {
ipc := "https://kovan.infura.io/J5Vd2fRtGsw0zZ0Ov3BL"
rpcClient, ethClient, err := getClients(ipc)
Expect(err).NotTo(HaveOccurred())
blockChain, err = getBlockChain(rpcClient, ethClient)

1
pkg/transformers/integration_tests/vat_grab.go
View File

@ -31,7 +31,6 @@ var _ = Describe("Vat Grab Transformer", func() {
config.StartingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber
ipc := "https://kovan.infura.io/J5Vd2fRtGsw0zZ0Ov3BL"
rpcClient, ethClient, err := getClients(ipc)
Expect(err).NotTo(HaveOccurred())
blockchain, err := getBlockChain(rpcClient, ethClient)

59
pkg/transformers/integration_tests/vat_slip.go Normal file
View File

@ -0,0 +1,59 @@
package integration_tests
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/vat_slip"
"github.com/vulcanize/vulcanizedb/test_config"
)
var _ = Describe("Vat slip transformer", func() {
var (
db *postgres.DB
blockChain core.BlockChain
)
BeforeEach(func() {
rpcClient, ethClient, err := getClients(ipc)
Expect(err).NotTo(HaveOccurred())
blockChain, err = getBlockChain(rpcClient, ethClient)
Expect(err).NotTo(HaveOccurred())
db = test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
err = persistHeader(rpcClient, db, 8953655)
Expect(err).NotTo(HaveOccurred())
})
It("persists vat slip event", func(done Done) {
config := shared.TransformerConfig{
ContractAddresses: []string{"0xcd726790550afcd77e9a7a47e86a3f9010af126b"},
StartingBlockNumber: 8953655,
EndingBlockNumber: 8953655,
}
transformerInitializer := vat_slip.VatSlipTransformerInitializer{Config: config}
transformer := transformerInitializer.NewVatSlipTransformer(db, blockChain)
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
var headerID int64
err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, config.StartingBlockNumber)
Expect(err).NotTo(HaveOccurred())
var model vat_slip.VatSlipModel
err = db.Get(&model, `SELECT ilk, guy, rad, tx_idx FROM maker.vat_slip WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(model.Ilk).To(Equal("ETH"))
Expect(model.Guy).To(Equal("0xDA15dCE70ab462E66779f23ee14F21d993789eE3"))
Expect(model.Rad).To(Equal("100000000000000000000000000000000000000000000000"))
Expect(model.TransactionIndex).To(Equal(uint(0)))
var headerChecked bool
err = db.Get(&headerChecked, `SELECT vat_slip_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue())
close(done)
}, 15)
})

1
pkg/transformers/integration_tests/vat_tune.go
View File

@ -31,7 +31,6 @@ var _ = Describe("VatTune Transformer", func() {
config.StartingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber
ipc := "https://kovan.infura.io/J5Vd2fRtGsw0zZ0Ov3BL"
rpcClient, ethClient, err := getClients(ipc)
Expect(err).NotTo(HaveOccurred())
blockchain, err := getBlockChain(rpcClient, ethClient)

2
pkg/transformers/shared/constants.go
View File

@ -60,6 +60,7 @@ var (
vatInitMethod = GetSolidityMethodSignature(VatABI, "init")
vatMoveMethod = GetSolidityMethodSignature(VatABI, "move")
vatFoldMethod = GetSolidityMethodSignature(VatABI, "fold")
vatSlipMethod = GetSolidityMethodSignature(VatABI, "slip")
vatTollMethod = GetSolidityMethodSignature(VatABI, "toll")
vatTuneMethod = GetSolidityMethodSignature(VatABI, "tune")
vatFluxMethod = GetSolidityMethodSignature(VatABI, "flux")
@ -87,6 +88,7 @@ var (
VatInitSignature = GetLogNoteSignature(vatInitMethod)
VatMoveSignature = GetLogNoteSignature(vatMoveMethod)
VatFoldSignature = GetLogNoteSignature(vatFoldMethod)
VatSlipSignature = GetLogNoteSignature(vatSlipMethod)
VatTollSignature = GetLogNoteSignature(vatTollMethod)
VatTuneSignature = GetLogNoteSignature(vatTuneMethod)
VatFluxSignature = GetLogNoteSignature(vatFluxMethod)

8
pkg/transformers/shared/event_signature_generator_test.go
View File

@ -160,6 +160,13 @@ var _ = Describe("Event signature generator", func() {
Expect(expected).To(Equal(actual))
})
It("gets the vat slip method signature", func() {
expected := "slip(bytes32,bytes32,int256)"
actual := shared.GetSolidityMethodSignature(shared.VatABI, "slip")
Expect(expected).To(Equal(actual))
})
})
Describe("it handles events", func() {
@ -196,7 +203,6 @@ var _ = Describe("Event signature generator", func() {
actual := shared.GetSolidityMethodSignature(shared.MedianizerABI, "LogValue")
Expect(expected).To(Equal(actual))
})
})
})

35
pkg/transformers/test_data/mocks/vat_slip/converter.go Normal file
View File

@ -0,0 +1,35 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package vat_slip
import (
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
"github.com/vulcanize/vulcanizedb/pkg/transformers/vat_slip"
)
type MockVatSlipConverter struct {
err error
PassedLogs []types.Log
}
func (converter *MockVatSlipConverter) ToModels(ethLogs []types.Log) ([]vat_slip.VatSlipModel, error) {
converter.PassedLogs = ethLogs
return []vat_slip.VatSlipModel{test_data.VatSlipModel}, converter.err
}
func (converter *MockVatSlipConverter) SetConverterError(e error) {
converter.err = e
}

71
pkg/transformers/test_data/mocks/vat_slip/repository.go Normal file
View File

@ -0,0 +1,71 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package vat_slip
import (
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/transformers/vat_slip"
)
type MockVatSlipRepository struct {
createError error
markHeaderCheckedError error
markHeaderCheckedPassedHeaderID int64
missingHeadersError error
missingHeaders []core.Header
PassedStartingBlockNumber int64
PassedEndingBlockNumber int64
PassedHeaderID int64
PassedModels []vat_slip.VatSlipModel
}
func (repository *MockVatSlipRepository) Create(headerID int64, models []vat_slip.VatSlipModel) error {
repository.PassedHeaderID = headerID
repository.PassedModels = models
return repository.createError
}
func (repository *MockVatSlipRepository) MarkHeaderChecked(headerID int64) error {
repository.markHeaderCheckedPassedHeaderID = headerID
return repository.markHeaderCheckedError
}
func (repository *MockVatSlipRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
repository.PassedStartingBlockNumber = startingBlockNumber
repository.PassedEndingBlockNumber = endingBlockNumber
return repository.missingHeaders, repository.missingHeadersError
}
func (repository *MockVatSlipRepository) SetCreateError(e error) {
repository.createError = e
}
func (repository *MockVatSlipRepository) SetMarkHeaderCheckedErr(e error) {
repository.markHeaderCheckedError = e
}
func (repository *MockVatSlipRepository) SetMissingHeadersErr(e error) {
repository.missingHeadersError = e
}
func (repository *MockVatSlipRepository) SetMissingHeaders(headers []core.Header) {
repository.missingHeaders = headers
}
func (repository *MockVatSlipRepository) AssertMarkHeaderCheckedCalledWith(i int64) {
Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i))
}

52
pkg/transformers/test_data/vat_slip.go Normal file
View File

@ -0,0 +1,52 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package test_data
import (
"encoding/json"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/vat_slip"
)
var EthVatSlipLog = types.Log{
Address: common.HexToAddress(shared.VatContractAddress),
Topics: []common.Hash{
common.HexToHash("0x42066cbb00000000000000000000000000000000000000000000000000000000"),
common.HexToHash("0x66616b6520696c6b000000000000000000000000000000000000000000000000"),
common.HexToHash("0x0000000000000000000000007d7bee5fcfd8028cf7b00876c5b1421c800561a6"),
common.HexToHash("0x000000000000000000000000000000000000000000000000000000003ade68b1"),
},
Data: hexutil.MustDecode("0x0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000006442066cbb66616b6520696c6b0000000000000000000000000000000000000000000000000000000000000000000000007d7bee5fcfd8028cf7b00876c5b1421c800561a6000000000000000000000000000000000000000000000000000000003ade68b1"),
BlockNumber: 10,
TxHash: common.HexToHash("0xb114ba306c80c86d51bdbf4a5ac8ed151020cd81b70cfa1dc9822f4a1f73930b"),
TxIndex: 3,
BlockHash: common.HexToHash("0x34b7e5ddb3be73257a5a0087f10b8bf68d4df5c8831ec04c63ecae4094de72ad"),
Index: 0,
Removed: false,
}
var rawVatSlipLog, _ = json.Marshal(EthVatSlipLog)
var VatSlipModel = vat_slip.VatSlipModel{
Ilk: "fake ilk",
Guy: common.HexToAddress("0x7d7bEe5fCfD8028cf7b00876C5b1421c800561A6").String(),
Rad: "987654321",
TransactionIndex: EthVatSlipLog.TxIndex,
Raw: rawVatSlipLog,
}

3
pkg/transformers/transformers.go
View File

@ -43,6 +43,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/transformers/vat_heal"
"github.com/vulcanize/vulcanizedb/pkg/transformers/vat_init"
"github.com/vulcanize/vulcanizedb/pkg/transformers/vat_move"
"github.com/vulcanize/vulcanizedb/pkg/transformers/vat_slip"
"github.com/vulcanize/vulcanizedb/pkg/transformers/vat_toll"
"github.com/vulcanize/vulcanizedb/pkg/transformers/vat_tune"
)
@ -74,6 +75,7 @@ var (
VatMoveTransformerInitializer = vat_move.VatMoveTransformerInitializer{Config: vat_move.VatMoveConfig}.NewVatMoveTransformer
VatHealTransformerInitializer = vat_heal.VatHealTransformerInitializer{Config: vat_heal.VatHealConfig}.NewVatHealTransformer
VatFoldTransformerInitializer = vat_fold.VatFoldTransformerInitializer{Config: vat_fold.VatFoldConfig}.NewVatFoldTransformer
VatSlipTransformerInitializer = vat_slip.VatSlipTransformerInitializer{Config: vat_slip.VatSlipConfig}.NewVatSlipTransformer
VatTollTransformerInitializer = vat_toll.VatTollTransformerInitializer{Config: vat_toll.VatTollConfig}.NewVatTollTransformer
VatTuneTransformerInitializer = vat_tune.VatTuneTransformerInitializer{Config: vat_tune.VatTuneConfig}.NewVatTuneTransformer
VatFluxTransformerInitializer = vat_flux.VatFluxTransformerInitializer{Config: vat_flux.VatFluxConfig}.NewVatFluxTransformer
@ -104,6 +106,7 @@ func TransformerInitializers() []shared.TransformerInitializer {
VatMoveTransformerInitializer,
VatHealTransformerInitializer,
VatFoldTransformerInitializer,
VatSlipTransformerInitializer,
VatTollTransformerInitializer,
VatTuneTransformerInitializer,
VatFluxTransformerInitializer,

11
pkg/transformers/vat_slip/config.go Normal file
View File

@ -0,0 +1,11 @@
package vat_slip
import "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
var VatSlipConfig = shared.TransformerConfig{
ContractAddresses: []string{shared.VatContractAddress},
ContractAbi: shared.VatABI,
Topics: []string{shared.VatSlipSignature},
StartingBlockNumber: 0,
EndingBlockNumber: 10000000,
}

64
pkg/transformers/vat_slip/converter.go Normal file
View File

@ -0,0 +1,64 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package vat_slip
import (
"bytes"
"encoding/json"
"errors"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
)
type Converter interface {
ToModels(ethLogs []types.Log) ([]VatSlipModel, error)
}
type VatSlipConverter struct{}
func (VatSlipConverter) ToModels(ethLogs []types.Log) ([]VatSlipModel, error) {
var models []VatSlipModel
for _, ethLog := range ethLogs {
err := verifyLog(ethLog)
if err != nil {
return nil, err
}
ilk := string(bytes.Trim(ethLog.Topics[1].Bytes(), "\x00"))
guy := common.BytesToAddress(ethLog.Topics[2].Bytes())
rad := ethLog.Topics[3].Big()
raw, err := json.Marshal(ethLog)
if err != nil {
return nil, err
}
model := VatSlipModel{
Ilk: ilk,
Guy: guy.String(),
Rad: rad.String(),
TransactionIndex: ethLog.TxIndex,
Raw: raw,
}
models = append(models, model)
}
return models, nil
}
func verifyLog(log types.Log) error {
numTopicInValidLog := 4
if len(log.Topics) < numTopicInValidLog {
return errors.New("log missing topics")
}
return nil
}

47
pkg/transformers/vat_slip/converter_test.go Normal file
View File

@ -0,0 +1,47 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package vat_slip_test
import (
"github.com/ethereum/go-ethereum/core/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
"github.com/vulcanize/vulcanizedb/pkg/transformers/vat_slip"
)
var _ = Describe("Vat slip converter", func() {
It("returns err if log is missing topics", func() {
converter := vat_slip.VatSlipConverter{}
badLog := types.Log{
Data: []byte{1, 1, 1, 1, 1},
}
_, err := converter.ToModels([]types.Log{badLog})
Expect(err).To(HaveOccurred())
})
It("converts a log to an model", func() {
converter := vat_slip.VatSlipConverter{}
models, err := converter.ToModels([]types.Log{test_data.EthVatSlipLog})
Expect(err).NotTo(HaveOccurred())
Expect(len(models)).To(Equal(1))
Expect(models[0]).To(Equal(test_data.VatSlipModel))
})
})

23
pkg/transformers/vat_slip/model.go Normal file
View File

@ -0,0 +1,23 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package vat_slip
type VatSlipModel struct {
Ilk string
Guy string
Rad string
TransactionIndex uint `db:"tx_idx"`
Raw []byte `db:"raw_log"`
}

74
pkg/transformers/vat_slip/repository.go Normal file
View File

@ -0,0 +1,74 @@
package vat_slip
import (
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
type Repository interface {
Create(headerID int64, models []VatSlipModel) error
MarkHeaderChecked(headerID int64) error
MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
}
type VatSlipRepository struct {
db *postgres.DB
}
func NewVatSlipRepository(db *postgres.DB) VatSlipRepository {
return VatSlipRepository{
db: db,
}
}
func (repository VatSlipRepository) Create(headerID int64, models []VatSlipModel) error {
tx, err := repository.db.Begin()
if err != nil {
return err
}
for _, model := range models {
_, err = tx.Exec(
`INSERT into maker.vat_slip (header_id, ilk, guy, rad, tx_idx, raw_log)
VALUES($1, $2, $3, $4::NUMERIC, $5, $6)`,
headerID, model.Ilk, model.Guy, model.Rad, model.TransactionIndex, model.Raw,
)
if err != nil {
tx.Rollback()
return err
}
}
_, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, vat_slip_checked)
VALUES ($1, $2)
ON CONFLICT (header_id) DO
UPDATE SET vat_slip_checked = $2`, headerID, true)
if err != nil {
tx.Rollback()
return err
}
return tx.Commit()
}
func (repository VatSlipRepository) MarkHeaderChecked(headerID int64) error {
_, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, vat_slip_checked)
VALUES ($1, $2)
ON CONFLICT (header_id) DO
UPDATE SET vat_slip_checked = $2`, headerID, true)
return err
}
func (repository VatSlipRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
var result []core.Header
err := repository.db.Select(
&result,
`SELECT headers.id, headers.block_number FROM headers
LEFT JOIN checked_headers on headers.id = header_id
WHERE (header_id ISNULL OR vat_slip_checked IS FALSE)
AND headers.block_number >= $1
AND headers.block_number <= $2
AND headers.eth_node_fingerprint = $3`,
startingBlockNumber,
endingBlockNumber,
repository.db.Node.ID,
)
return result, err
}

178
pkg/transformers/vat_slip/repository_test.go Normal file
View File

@ -0,0 +1,178 @@
package vat_slip_test
import (
"database/sql"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
"github.com/vulcanize/vulcanizedb/pkg/transformers/vat_slip"
"github.com/vulcanize/vulcanizedb/test_config"
)
var _ = Describe("Vat slip repository", func() {
var (
db *postgres.DB
vatSlipRepository vat_slip.Repository
err error
headerRepository datastore.HeaderRepository
)
BeforeEach(func() {
db = test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository = repositories.NewHeaderRepository(db)
vatSlipRepository = vat_slip.NewVatSlipRepository(db)
})
Describe("Create", func() {
var headerID int64
BeforeEach(func() {
headerID, err = headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
Expect(err).NotTo(HaveOccurred())
err = vatSlipRepository.Create(headerID, []vat_slip.VatSlipModel{test_data.VatSlipModel})
Expect(err).NotTo(HaveOccurred())
})
It("adds a vat slip event", func() {
var dbVatSlip vat_slip.VatSlipModel
err = db.Get(&dbVatSlip, `SELECT ilk, guy, rad, tx_idx, raw_log FROM maker.vat_slip WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(dbVatSlip.Ilk).To(Equal(test_data.VatSlipModel.Ilk))
Expect(dbVatSlip.Guy).To(Equal(test_data.VatSlipModel.Guy))
Expect(dbVatSlip.Rad).To(Equal(test_data.VatSlipModel.Rad))
Expect(dbVatSlip.TransactionIndex).To(Equal(test_data.VatSlipModel.TransactionIndex))
Expect(dbVatSlip.Raw).To(MatchJSON(test_data.VatSlipModel.Raw))
})
It("marks header as checked for logs", func() {
var headerChecked bool
err = db.Get(&headerChecked, `SELECT vat_slip_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue())
})
It("does not duplicate vat slip events", func() {
err = vatSlipRepository.Create(headerID, []vat_slip.VatSlipModel{test_data.VatSlipModel})
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
})
It("removes vat slip if corresponding header is deleted", func() {
_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
var dbVatSlip vat_slip.VatSlipModel
err = db.Get(&dbVatSlip, `SELECT ilk, guy, rad, tx_idx, raw_log FROM maker.vat_slip WHERE header_id = $1`, headerID)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(sql.ErrNoRows))
})
})
Describe("MarkHeaderChecked", func() {
var headerID int64
BeforeEach(func() {
headerID, err = headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
Expect(err).NotTo(HaveOccurred())
})
It("creates a row for a new headerID", func() {
err = vatSlipRepository.MarkHeaderChecked(headerID)
Expect(err).NotTo(HaveOccurred())
var headerChecked bool
err = db.Get(&headerChecked, `SELECT vat_slip_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue())
})
It("updates row when headerID already exists", func() {
_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID)
err = vatSlipRepository.MarkHeaderChecked(headerID)
Expect(err).NotTo(HaveOccurred())
var headerChecked bool
err = db.Get(&headerChecked, `SELECT vat_slip_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue())
})
})
Describe("MissingHeaders", func() {
var (
startingBlock, endingBlock, vatSlipBlock int64
blockNumbers, headerIDs []int64
)
BeforeEach(func() {
startingBlock = GinkgoRandomSeed()
vatSlipBlock = startingBlock + 1
endingBlock = startingBlock + 2
blockNumbers = []int64{startingBlock, vatSlipBlock, endingBlock, endingBlock + 1}
headerIDs = []int64{}
for _, n := range blockNumbers {
headerID, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(n))
Expect(err).NotTo(HaveOccurred())
headerIDs = append(headerIDs, headerID)
}
})
It("returns headers that haven't been checked", func() {
err := vatSlipRepository.MarkHeaderChecked(headerIDs[1])
Expect(err).NotTo(HaveOccurred())
headers, err := vatSlipRepository.MissingHeaders(startingBlock, endingBlock)
Expect(err).NotTo(HaveOccurred())
Expect(len(headers)).To(Equal(2))
Expect(headers[0].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock)))
Expect(headers[1].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock)))
})
It("only treats headers as checked if vat slip logs have been checked", func() {
_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
Expect(err).NotTo(HaveOccurred())
headers, err := vatSlipRepository.MissingHeaders(startingBlock, endingBlock)
Expect(err).NotTo(HaveOccurred())
Expect(len(headers)).To(Equal(3))
Expect(headers[0].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(vatSlipBlock)))
Expect(headers[1].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(vatSlipBlock)))
Expect(headers[2].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(vatSlipBlock)))
})
It("only returns headers associated with the current node", func() {
dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
for _, n := range blockNumbers {
_, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n))
Expect(err).NotTo(HaveOccurred())
}
vatSlipRepositoryTwo := vat_slip.NewVatSlipRepository(dbTwo)
err := vatSlipRepository.MarkHeaderChecked(headerIDs[0])
Expect(err).NotTo(HaveOccurred())
nodeOneMissingHeaders, err := vatSlipRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])
Expect(err).NotTo(HaveOccurred())
Expect(len(nodeOneMissingHeaders)).To(Equal(len(blockNumbers) - 1))
nodeTwoMissingHeaders, err := vatSlipRepositoryTwo.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])
Expect(err).NotTo(HaveOccurred())
Expect(len(nodeTwoMissingHeaders)).To(Equal(len(blockNumbers)))
})
})
})

64
pkg/transformers/vat_slip/transformer.go Normal file
View File

@ -0,0 +1,64 @@
package vat_slip
import (
"log"
"github.com/ethereum/go-ethereum/common"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
)
type VatSlipTransformer struct {
Config shared.TransformerConfig
Converter Converter
Fetcher shared.LogFetcher
Repository Repository
}
type VatSlipTransformerInitializer struct {
Config shared.TransformerConfig
}
func (initializer VatSlipTransformerInitializer) NewVatSlipTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer {
converter := VatSlipConverter{}
fetcher := shared.NewFetcher(blockChain)
repository := NewVatSlipRepository(db)
return VatSlipTransformer{
Config: initializer.Config,
Converter: converter,
Fetcher: fetcher,
Repository: repository,
}
}
func (transformer VatSlipTransformer) Execute() error {
missingHeaders, err := transformer.Repository.MissingHeaders(transformer.Config.StartingBlockNumber, transformer.Config.EndingBlockNumber)
if err != nil {
return err
}
log.Printf("Fetching vat slip event logs for %d headers \n", len(missingHeaders))
for _, header := range missingHeaders {
topics := [][]common.Hash{{common.HexToHash(shared.VatSlipSignature)}}
matchingLogs, err := transformer.Fetcher.FetchLogs(VatSlipConfig.ContractAddresses, topics, header.BlockNumber)
if err != nil {
return err
}
if len(matchingLogs) < 1 {
err = transformer.Repository.MarkHeaderChecked(header.Id)
if err != nil {
return err
}
}
models, err := transformer.Converter.ToModels(matchingLogs)
if err != nil {
return err
}
err = transformer.Repository.Create(header.Id, models)
if err != nil {
return err
}
}
return nil
}

143
pkg/transformers/vat_slip/transformer_test.go Normal file
View File

@ -0,0 +1,143 @@
package vat_slip_test
import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks"
vat_slip_mocks "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks/vat_slip"
"github.com/vulcanize/vulcanizedb/pkg/transformers/vat_slip"
)
var _ = Describe("Vat slip transformer", func() {
var (
config shared.TransformerConfig
converter *vat_slip_mocks.MockVatSlipConverter
fetcher *mocks.MockLogFetcher
repository *vat_slip_mocks.MockVatSlipRepository
transformer vat_slip.VatSlipTransformer
)
BeforeEach(func() {
config = vat_slip.VatSlipConfig
converter = &vat_slip_mocks.MockVatSlipConverter{}
fetcher = &mocks.MockLogFetcher{}
repository = &vat_slip_mocks.MockVatSlipRepository{}
transformer = vat_slip.VatSlipTransformer{
Config: config,
Converter: converter,
Fetcher: fetcher,
Repository: repository,
}
})
It("gets missing headers for block numbers specified in config", func() {
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(repository.PassedStartingBlockNumber).To(Equal(vat_slip.VatSlipConfig.StartingBlockNumber))
Expect(repository.PassedEndingBlockNumber).To(Equal(vat_slip.VatSlipConfig.EndingBlockNumber))
})
It("returns error if repository returns error for missing headers", func() {
repository.SetMissingHeadersErr(fakes.FakeError)
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("fetches logs for missing headers", func() {
headerOne := core.Header{BlockNumber: GinkgoRandomSeed()}
headerTwo := core.Header{BlockNumber: GinkgoRandomSeed()}
repository.SetMissingHeaders([]core.Header{headerOne, headerTwo})
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(fetcher.FetchedBlocks).To(Equal([]int64{headerOne.BlockNumber, headerTwo.BlockNumber}))
Expect(fetcher.FetchedContractAddresses).To(Equal([][]string{vat_slip.VatSlipConfig.ContractAddresses, vat_slip.VatSlipConfig.ContractAddresses}))
Expect(fetcher.FetchedTopics).To(Equal([][]common.Hash{{common.HexToHash(shared.VatSlipSignature)}}))
})
It("returns error if fetcher returns error", func() {
fetcher.SetFetcherError(fakes.FakeError)
repository.SetMissingHeaders([]core.Header{{BlockNumber: GinkgoRandomSeed()}})
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("marks header checked if no logs returned", func() {
headerID := GinkgoRandomSeed()
repository.SetMissingHeaders([]core.Header{{Id: headerID}})
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
repository.AssertMarkHeaderCheckedCalledWith(headerID)
})
It("returns error if marking header checked returns err", func() {
repository.SetMissingHeaders([]core.Header{{Id: GinkgoRandomSeed()}})
repository.SetMarkHeaderCheckedErr(fakes.FakeError)
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("converts matching logs", func() {
fetcher.SetFetchedLogs([]types.Log{test_data.EthVatSlipLog})
repository.SetMissingHeaders([]core.Header{{BlockNumber: GinkgoRandomSeed()}})
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(converter.PassedLogs).To(Equal([]types.Log{test_data.EthVatSlipLog}))
})
It("returns error if converter returns error", func() {
converter.SetConverterError(fakes.FakeError)
fetcher.SetFetchedLogs([]types.Log{test_data.EthVatSlipLog})
repository.SetMissingHeaders([]core.Header{{BlockNumber: GinkgoRandomSeed()}})
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("persists vat slip model", func() {
fetcher.SetFetchedLogs([]types.Log{test_data.EthVatSlipLog})
fakeHeader := core.Header{BlockNumber: GinkgoRandomSeed(), Id: GinkgoRandomSeed()}
repository.SetMissingHeaders([]core.Header{fakeHeader})
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id))
Expect(repository.PassedModels).To(Equal([]vat_slip.VatSlipModel{test_data.VatSlipModel}))
})
It("returns error if repository returns error for create", func() {
fetcher.SetFetchedLogs([]types.Log{test_data.EthVatSlipLog})
repository.SetMissingHeaders([]core.Header{{BlockNumber: GinkgoRandomSeed(), Id: GinkgoRandomSeed()}})
repository.SetCreateError(fakes.FakeError)
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
})

33
pkg/transformers/vat_slip/vat_slip_suite_test.go Normal file
View File

@ -0,0 +1,33 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package vat_slip_test
import (
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"io/ioutil"
"log"
)
func TestVatSlip(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "VatSlip Suite")
}
var _ = BeforeSuite(func() {
log.SetOutput(ioutil.Discard)
})

1
test_config/test_config.go
View File

@ -101,6 +101,7 @@ func CleanTestDB(db *postgres.DB) {
db.MustExec("DELETE FROM maker.vat_init")
db.MustExec("DELETE FROM maker.vat_move")
db.MustExec("DELETE FROM maker.vat_fold")
db.MustExec("DELETE FROM maker.vat_slip")
db.MustExec("DELETE FROM maker.vat_toll")
db.MustExec("DELETE FROM maker.vat_tune")
db.MustExec("DELETE FROM maker.vat_flux")