deep-stack/ipld-eth-server
4
0
Fork 0
forked from cerc-io/ipld-eth-server
Code Pull Requests Activity

Add raw log to Frob record in DB

Browse Source 
- Enables parsing additional values off the log if necessary
- Also update Frob to reflect DSS changes (lad => urn)
- Move Frob constants to shared directory
This commit is contained in:
Rob Mulholand
2018-09-04 16:11:54 -05:00
parent db4b891398
commit 8f5bc0a956
19 changed files with 167 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/transformers/frob/config.go
+2 -2
View File
@@ -18,8 +18,8 @@ import "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
var FrobConfig = shared.TransformerConfig{
ContractAddress: "0xff3f2400f1600f3f493a9a92704a29b96795af1a", //this is a temporary address deployed locally
ContractAbi: FrobABI,
Topics: []string{FrobEventSignature},
ContractAbi: shared.PitABI,
Topics: []string{shared.FrobSignature},
StartingBlockNumber: 0,
EndingBlockNumber: 100,
}
pkg/transformers/frob/constants.go
-20
View File
@@ -1,20 +0,0 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package frob
var (
FrobABI = `[{"constant":true,"inputs":[],"name":"vat","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"live","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"drip","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"Line","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"","type":"address"}],"name":"wards","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"","type":"bytes32"}],"name":"ilks","outputs":[{"name":"spot","type":"uint256"},{"name":"line","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"inputs":[{"name":"vat_","type":"address"}],"payable":false,"stateMutability":"nonpayable","type":"constructor"},{"anonymous":false,"inputs":[{"indexed":true,"name":"ilk","type":"bytes32"},{"indexed":true,"name":"lad","type":"bytes32"},{"indexed":false,"name":"ink","type":"uint256"},{"indexed":false,"name":"art","type":"uint256"},{"indexed":false,"name":"dink","type":"int256"},{"indexed":false,"name":"dart","type":"int256"},{"indexed":false,"name":"iArt","type":"uint256"}],"name":"Frob","type":"event"},{"anonymous":true,"inputs":[{"indexed":true,"name":"sig","type":"bytes4"},{"indexed":true,"name":"guy","type":"address"},{"indexed":true,"name":"foo","type":"bytes32"},{"indexed":true,"name":"bar","type":"bytes32"},{"indexed":false,"name":"wad","type":"uint256"},{"indexed":false,"name":"fax","type":"bytes"}],"name":"LogNote","type":"event"},{"constant":false,"inputs":[{"name":"guy","type":"address"}],"name":"rely","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"guy","type":"address"}],"name":"deny","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"ilk","type":"bytes32"},{"name":"what","type":"bytes32"},{"name":"data","type":"uint256"}],"name":"file","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"what","type":"bytes32"},{"name":"data","type":"uint256"}],"name":"file","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"what","type":"bytes32"},{"name":"data","type":"address"}],"name":"file","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"ilk","type":"bytes32"},{"name":"dink","type":"int256"},{"name":"dart","type":"int256"}],"name":"frob","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"}]`
FrobEventSignature = "0x6cedf1d3a466a3d6bab04887b1642177bf6dbf1daa737c2e8f639cd0b020d9d0"
)
pkg/transformers/frob/converter.go
+19 -10
View File
@@ -19,12 +19,13 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"encoding/json"
"github.com/vulcanize/vulcanizedb/pkg/geth"
)
type Converter interface {
ToEntity(contractAddress string, contractAbi string, ethLog types.Log) (FrobEntity, error)
ToModel(flipKick FrobEntity) FrobModel
ToModel(flipKick FrobEntity) (FrobModel, error)
}
type FrobConverter struct {
@@ -39,17 +40,25 @@ func (FrobConverter) ToEntity(contractAddress string, contractAbi string, ethLog
}
contract := bind.NewBoundContract(address, abi, nil, nil, nil)
err = contract.UnpackLog(&entity, "Frob", ethLog)
entity.TransactionIndex = ethLog.TxIndex
entity.Raw = ethLog
return entity, err
}
func (FrobConverter) ToModel(frob FrobEntity) FrobModel {
return FrobModel{
Ilk: frob.Ilk[:],
Lad: frob.Lad[:],
Dink: frob.Dink.String(),
Dart: frob.Dart.String(),
Ink: frob.Ink.String(),
Art: frob.Art.String(),
IArt: frob.IArt.String(),
func (FrobConverter) ToModel(frob FrobEntity) (FrobModel, error) {
rawLog, err := json.Marshal(frob.Raw)
if err != nil {
return FrobModel{}, err
}
return FrobModel{
Ilk: frob.Ilk[:],
Urn: frob.Urn[:],
Ink: frob.Ink.String(),
Art: frob.Art.String(),
Dink: frob.Dink.String(),
Dart: frob.Dart.String(),
IArt: frob.IArt.String(),
TransactionIndex: frob.TransactionIndex,
Raw: rawLog,
}, nil
}
pkg/transformers/frob/converter_test.go
+4 -2
View File
@@ -19,6 +19,7 @@ import (
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/transformers/frob"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
)
@@ -26,7 +27,7 @@ var _ = Describe("Frob converter", func() {
It("converts a log to an entity", func() {
converter := frob.FrobConverter{}
entity, err := converter.ToEntity(test_data.TemporaryFrobAddress, frob.FrobABI, test_data.EthFrobLog)
entity, err := converter.ToEntity(test_data.TemporaryFrobAddress, shared.PitABI, test_data.EthFrobLog)
Expect(err).NotTo(HaveOccurred())
Expect(entity).To(Equal(test_data.FrobEntity))
@@ -35,8 +36,9 @@ var _ = Describe("Frob converter", func() {
It("converts an entity to a model", func() {
converter := frob.FrobConverter{}
model := converter.ToModel(test_data.FrobEntity)
model, err := converter.ToModel(test_data.FrobEntity)
Expect(err).NotTo(HaveOccurred())
Expect(model).To(Equal(test_data.FrobModel))
})
})
pkg/transformers/frob/entity.go
+11 -7
View File
@@ -16,14 +16,18 @@ package frob
import (
"math/big"
"github.com/ethereum/go-ethereum/core/types"
)
type FrobEntity struct {
Ilk [32]byte
Lad [32]byte
Ink *big.Int
Art *big.Int
Dink *big.Int
Dart *big.Int
IArt *big.Int
Ilk [32]byte
Urn [32]byte
Ink *big.Int
Art *big.Int
Dink *big.Int
Dart *big.Int
IArt *big.Int
TransactionIndex uint
Raw types.Log
}
pkg/transformers/frob/integration_test.go
+3 -3
View File
@@ -46,7 +46,7 @@ var _ = Describe("Integration tests", func() {
transactionConverter := vRpc.NewRpcTransactionConverter(ethClient)
realBlockChain := geth.NewBlockChain(blockChainClient, realNode, transactionConverter)
realFetcher := shared.NewFetcher(realBlockChain)
topic0 := common.HexToHash(frob.FrobEventSignature)
topic0 := common.HexToHash(shared.FrobSignature)
topics := [][]common.Hash{{topic0}}
result, err := realFetcher.FetchLogs(test_data.TemporaryFrobAddress, topics, int64(12))
@@ -62,7 +62,7 @@ var _ = Describe("Integration tests", func() {
It("unpacks an event log", func() {
address := common.HexToAddress(test_data.TemporaryFrobAddress)
abi, err := geth.ParseAbi(frob.FrobABI)
abi, err := geth.ParseAbi(shared.PitABI)
Expect(err).NotTo(HaveOccurred())
contract := bind.NewBoundContract(address, abi, nil, nil, nil)
@@ -78,6 +78,6 @@ var _ = Describe("Integration tests", func() {
Expect(entity.IArt).To(Equal(expectedEntity.IArt))
Expect(entity.Ilk).To(Equal(expectedEntity.Ilk))
Expect(entity.Ink).To(Equal(expectedEntity.Ink))
Expect(entity.Lad).To(Equal(expectedEntity.Lad))
Expect(entity.Urn).To(Equal(expectedEntity.Urn))
})
})
pkg/transformers/frob/model.go
+9 -7
View File
@@ -15,11 +15,13 @@
package frob
type FrobModel struct {
Ilk []byte
Lad []byte
Ink string
Art string
Dink string
Dart string
IArt string
Ilk []byte
Urn []byte
Ink string
Art string
Dink string
Dart string
IArt string
TransactionIndex uint `db:"tx_idx"`
Raw []byte `db:"raw_log"`
}
pkg/transformers/frob/repository.go
+5 -5
View File
@@ -20,7 +20,7 @@ import (
)
type Repository interface {
Create(headerID int64, transactionIndex uint, model FrobModel) error
Create(headerID int64, model FrobModel) error
MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
}
@@ -32,10 +32,10 @@ func NewFrobRepository(db *postgres.DB) FrobRepository {
return FrobRepository{db: db}
}
func (repository FrobRepository) Create(headerID int64, transactionIndex uint, model FrobModel) error {
_, err := repository.db.Exec(`INSERT INTO maker.frob (header_id, tx_idx, art, dart, dink, iart, ilk, ink, lad)
VALUES($1, $2, $3::NUMERIC, $4::NUMERIC, $5::NUMERIC, $6::NUMERIC, $7, $8::NUMERIC, $9)`,
headerID, transactionIndex, model.Art, model.Dart, model.Dink, model.IArt, model.Ilk, model.Ink, model.Lad)
func (repository FrobRepository) Create(headerID int64, model FrobModel) error {
_, err := repository.db.Exec(`INSERT INTO maker.frob (header_id, art, dart, dink, iart, ilk, ink, urn, raw_log, tx_idx)
VALUES($1, $2::NUMERIC, $3::NUMERIC, $4::NUMERIC, $5::NUMERIC, $6, $7::NUMERIC, $8, $9, $10)`,
headerID, model.Art, model.Dart, model.Dink, model.IArt, model.Ilk, model.Ink, model.Urn, model.Raw, model.TransactionIndex)
return err
}
pkg/transformers/frob/repository_test.go
+23 -21
View File
@@ -30,56 +30,61 @@ import (
var _ = Describe("Frob repository", func() {
Describe("Create", func() {
It("adds a frob", func() {
node := core.Node{}
db := test_config.NewTestDB(node)
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
frobRepository := frob.NewFrobRepository(db)
err = frobRepository.Create(headerID, 123, test_data.FrobModel)
err = frobRepository.Create(headerID, test_data.FrobModel)
Expect(err).NotTo(HaveOccurred())
var dbFrob frob.FrobModel
err = db.Get(&dbFrob, `SELECT art, dart, dink, iart, ilk, ink, lad FROM maker.frob WHERE header_id = $1`, headerID)
err = db.Get(&dbFrob, `SELECT art, dart, dink, iart, ilk, ink, urn, tx_idx, raw_log FROM maker.frob WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(dbFrob).To(Equal(test_data.FrobModel))
Expect(dbFrob.Ilk).To(Equal(test_data.FrobModel.Ilk))
Expect(dbFrob.Urn).To(Equal(test_data.FrobModel.Urn))
Expect(dbFrob.Ink).To(Equal(test_data.FrobModel.Ink))
Expect(dbFrob.Art).To(Equal(test_data.FrobModel.Art))
Expect(dbFrob.Dink).To(Equal(test_data.FrobModel.Dink))
Expect(dbFrob.Dart).To(Equal(test_data.FrobModel.Dart))
Expect(dbFrob.IArt).To(Equal(test_data.FrobModel.IArt))
Expect(dbFrob.TransactionIndex).To(Equal(test_data.FrobModel.TransactionIndex))
Expect(dbFrob.Raw).To(MatchJSON(test_data.FrobModel.Raw))
})
It("does not duplicate frob events", func() {
node := core.Node{}
db := test_config.NewTestDB(node)
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
frobRepository := frob.NewFrobRepository(db)
err = frobRepository.Create(headerID, 123, test_data.FrobModel)
err = frobRepository.Create(headerID, test_data.FrobModel)
Expect(err).NotTo(HaveOccurred())
err = frobRepository.Create(headerID, 123, test_data.FrobModel)
err = frobRepository.Create(headerID, test_data.FrobModel)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
})
It("removes frob if corresponding header is deleted", func() {
node := core.Node{}
db := test_config.NewTestDB(node)
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
frobRepository := frob.NewFrobRepository(db)
err = frobRepository.Create(headerID, 123, test_data.FrobModel)
err = frobRepository.Create(headerID, test_data.FrobModel)
Expect(err).NotTo(HaveOccurred())
_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
var dbFrob frob.FrobModel
err = db.Get(&dbFrob, `SELECT art, iart, ilk, ink, lad FROM maker.frob WHERE header_id = $1`, headerID)
err = db.Get(&dbFrob, `SELECT art, iart, ilk, ink, urn, tx_idx, raw_log FROM maker.frob WHERE header_id = $1`, headerID)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(sql.ErrNoRows))
})
@@ -87,8 +92,7 @@ var _ = Describe("Frob repository", func() {
Describe("MissingHeaders", func() {
It("returns headers with no associated frob event", func() {
node := core.Node{}
db := test_config.NewTestDB(node)
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
startingBlockNumber := int64(1)
@@ -102,7 +106,7 @@ var _ = Describe("Frob repository", func() {
Expect(err).NotTo(HaveOccurred())
}
frobRepository := frob.NewFrobRepository(db)
err := frobRepository.Create(headerIDs[1], 123, test_data.FrobModel)
err := frobRepository.Create(headerIDs[1], test_data.FrobModel)
Expect(err).NotTo(HaveOccurred())
headers, err := frobRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
@@ -114,13 +118,11 @@ var _ = Describe("Frob repository", func() {
})
It("only returns headers associated with the current node", func() {
nodeOne := core.Node{}
db := test_config.NewTestDB(nodeOne)
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
blockNumbers := []int64{1, 2, 3}
headerRepository := repositories.NewHeaderRepository(db)
nodeTwo := core.Node{ID: "second"}
dbTwo := test_config.NewTestDB(nodeTwo)
dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
var headerIDs []int64
for _, n := range blockNumbers {
@@ -132,7 +134,7 @@ var _ = Describe("Frob repository", func() {
}
frobRepository := frob.NewFrobRepository(db)
frobRepositoryTwo := frob.NewFrobRepository(dbTwo)
err := frobRepository.Create(headerIDs[0], 0, test_data.FrobModel)
err := frobRepository.Create(headerIDs[0], test_data.FrobModel)
Expect(err).NotTo(HaveOccurred())
nodeOneMissingHeaders, err := frobRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])
pkg/transformers/frob/transformer.go
+6 -3
View File
@@ -51,7 +51,7 @@ func (transformer FrobTransformer) Execute() error {
return err
}
for _, header := range missingHeaders {
topics := [][]common.Hash{{common.HexToHash(FrobEventSignature)}}
topics := [][]common.Hash{{common.HexToHash(shared.FrobSignature)}}
matchingLogs, err := transformer.Fetcher.FetchLogs(FrobConfig.ContractAddress, topics, header.BlockNumber)
if err != nil {
return err
@@ -61,8 +61,11 @@ func (transformer FrobTransformer) Execute() error {
if err != nil {
return err
}
model := transformer.Converter.ToModel(entity)
err = transformer.Repository.Create(header.Id, log.TxIndex, model)
model, err := transformer.Converter.ToModel(entity)
if err != nil {
return err
}
err = transformer.Repository.Create(header.Id, model)
if err != nil {
return err
}
pkg/transformers/frob/transformer_test.go
+41 -5
View File
@@ -23,6 +23,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/transformers/frob"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks"
frob_mocks "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks/frob"
@@ -75,7 +76,7 @@ var _ = Describe("Frob transformer", func() {
Expect(err).NotTo(HaveOccurred())
Expect(fetcher.FetchedBlocks).To(Equal([]int64{1, 2}))
Expect(fetcher.FetchedContractAddress).To(Equal(frob.FrobConfig.ContractAddress))
Expect(fetcher.FetchedTopics).To(Equal([][]common.Hash{{common.HexToHash(frob.FrobEventSignature)}}))
Expect(fetcher.FetchedTopics).To(Equal([][]common.Hash{{common.HexToHash(shared.FrobSignature)}}))
})
It("returns error if fetcher returns error", func() {
@@ -95,7 +96,7 @@ var _ = Describe("Frob transformer", func() {
Expect(err).To(MatchError(fakes.FakeError))
})
It("converts matching logs", func() {
It("converts matching logs to entity", func() {
converter := &frob_mocks.MockFrobConverter{}
fetcher := &mocks.MockLogFetcher{}
fetcher.SetFetchedLogs([]types.Log{test_data.EthFrobLog})
@@ -113,12 +114,48 @@ var _ = Describe("Frob transformer", func() {
Expect(converter.PassedContractAddress).To(Equal(frob.FrobConfig.ContractAddress))
Expect(converter.PassedContractABI).To(Equal(frob.FrobConfig.ContractAbi))
Expect(converter.PassedLog).To(Equal(test_data.EthFrobLog))
})
It("returns error if converting to entity returns error", func() {
converter := &frob_mocks.MockFrobConverter{}
converter.SetToEntityError(fakes.FakeError)
fetcher := &mocks.MockLogFetcher{}
fetcher.SetFetchedLogs([]types.Log{test_data.EthFrobLog})
repository := &frob_mocks.MockFrobRepository{}
repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}})
transformer := frob.FrobTransformer{
Fetcher: fetcher,
Converter: converter,
Repository: repository,
}
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("converts frob entity to model", func() {
converter := &frob_mocks.MockFrobConverter{}
fetcher := &mocks.MockLogFetcher{}
fetcher.SetFetchedLogs([]types.Log{test_data.EthFrobLog})
repository := &frob_mocks.MockFrobRepository{}
repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}})
transformer := frob.FrobTransformer{
Fetcher: fetcher,
Converter: converter,
Repository: repository,
}
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(converter.PassedEntity).To(Equal(test_data.FrobEntity))
})
It("returns error if converter returns error", func() {
It("returns error if converting to model returns error", func() {
converter := &frob_mocks.MockFrobConverter{}
converter.SetConverterError(fakes.FakeError)
converter.SetToModelError(fakes.FakeError)
fetcher := &mocks.MockLogFetcher{}
fetcher.SetFetchedLogs([]types.Log{test_data.EthFrobLog})
repository := &frob_mocks.MockFrobRepository{}
@@ -152,7 +189,6 @@ var _ = Describe("Frob transformer", func() {
Expect(err).NotTo(HaveOccurred())
Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id))
Expect(repository.PassedTransactionIndex).To(Equal(test_data.EthFrobLog.TxIndex))
Expect(repository.PassedFrobModel).To(Equal(test_data.FrobModel))
})