deep-stack/ipld-eth-server
4
0
Fork 0
forked from cerc-io/ipld-eth-server
Code Pull Requests Activity

Log checked headers for MCD events (#43)

Browse Source 
* Log checked headers for Bite events

* Log checked headers for Drip drip events

* Log checked headers for Drip file events

- mark headers checked for 3 implementations of file on the Drip contract

* Log checked headers for Frob frob events

* Log checked headers for Pit file events

- mark headers checked for 3 implementations of file on the Pit contract

* Log checked headers for Vat init events
This commit is contained in:
Rob Mulholand
2018-10-02 17:25:38 -05:00
committed by Elizabeth
parent 5a2d05b8d0
commit bf44ec614a
93 changed files with 2439 additions and 772 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/transformers/frob/converter.go
+38 -29
View File
@@ -16,7 +16,6 @@ package frob
import (
"github.com/ethereum/go-ethereum/accounts/abi/bind"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"encoding/json"
@@ -24,40 +23,50 @@ import (
)
type Converter interface {
ToEntity(contractAddress string, contractAbi string, ethLog types.Log) (FrobEntity, error)
ToModel(flipKick FrobEntity) (FrobModel, error)
ToEntities(contractAbi string, ethLogs []types.Log) ([]FrobEntity, error)
ToModels(entities []FrobEntity) ([]FrobModel, error)
}
type FrobConverter struct{}
func (FrobConverter) ToEntity(contractAddress string, contractAbi string, ethLog types.Log) (FrobEntity, error) {
entity := FrobEntity{}
address := common.HexToAddress(contractAddress)
abi, err := geth.ParseAbi(contractAbi)
if err != nil {
return entity, err
func (FrobConverter) ToEntities(contractAbi string, ethLogs []types.Log) ([]FrobEntity, error) {
var entities []FrobEntity
for _, ethLog := range ethLogs {
entity := FrobEntity{}
address := ethLog.Address
abi, err := geth.ParseAbi(contractAbi)
if err != nil {
return nil, err
}
contract := bind.NewBoundContract(address, abi, nil, nil, nil)
err = contract.UnpackLog(&entity, "Frob", ethLog)
entity.TransactionIndex = ethLog.TxIndex
entity.Raw = ethLog
entities = append(entities, entity)
}
contract := bind.NewBoundContract(address, abi, nil, nil, nil)
err = contract.UnpackLog(&entity, "Frob", ethLog)
entity.TransactionIndex = ethLog.TxIndex
entity.Raw = ethLog
return entity, err
return entities, nil
}
func (FrobConverter) ToModel(frob FrobEntity) (FrobModel, error) {
rawLog, err := json.Marshal(frob.Raw)
if err != nil {
return FrobModel{}, err
func (FrobConverter) ToModels(entities []FrobEntity) ([]FrobModel, error) {
var models []FrobModel
for _, entity := range entities {
rawLog, err := json.Marshal(entity.Raw)
if err != nil {
return nil, err
}
model := FrobModel{
Ilk: entity.Ilk[:],
Urn: entity.Urn[:],
Ink: entity.Ink.String(),
Art: entity.Art.String(),
Dink: entity.Dink.String(),
Dart: entity.Dart.String(),
IArt: entity.IArt.String(),
TransactionIndex: entity.TransactionIndex,
Raw: rawLog,
}
models = append(models, model)
}
return FrobModel{
Ilk: frob.Ilk[:],
Urn: frob.Urn[:],
Ink: frob.Ink.String(),
Art: frob.Art.String(),
Dink: frob.Dink.String(),
Dart: frob.Dart.String(),
IArt: frob.IArt.String(),
TransactionIndex: frob.TransactionIndex,
Raw: rawLog,
}, nil
return models, nil
}
pkg/transformers/frob/converter_test.go
+7 -4
View File
@@ -18,6 +18,7 @@ import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/transformers/frob"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
@@ -27,18 +28,20 @@ var _ = Describe("Frob converter", func() {
It("converts a log to an entity", func() {
converter := frob.FrobConverter{}
entity, err := converter.ToEntity(shared.PitContractAddress, shared.PitABI, test_data.EthFrobLog)
entities, err := converter.ToEntities(shared.PitABI, []types.Log{test_data.EthFrobLog})
Expect(err).NotTo(HaveOccurred())
Expect(entity).To(Equal(test_data.FrobEntity))
Expect(len(entities)).To(Equal(1))
Expect(entities[0]).To(Equal(test_data.FrobEntity))
})
It("converts an entity to a model", func() {
converter := frob.FrobConverter{}
model, err := converter.ToModel(test_data.FrobEntity)
models, err := converter.ToModels([]frob.FrobEntity{test_data.FrobEntity})
Expect(err).NotTo(HaveOccurred())
Expect(model).To(Equal(test_data.FrobModel))
Expect(len(models)).To(Equal(1))
Expect(models[0]).To(Equal(test_data.FrobModel))
})
})
pkg/transformers/frob/repository.go
+33 -6
View File
@@ -20,7 +20,8 @@ import (
)
type Repository interface {
Create(headerID int64, model FrobModel) error
Create(headerID int64, models []FrobModel) error
MarkHeaderChecked(headerID int64) error
MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
}
@@ -32,10 +33,36 @@ func NewFrobRepository(db *postgres.DB) FrobRepository {
return FrobRepository{db: db}
}
func (repository FrobRepository) Create(headerID int64, model FrobModel) error {
_, err := repository.db.Exec(`INSERT INTO maker.frob (header_id, art, dart, dink, iart, ilk, ink, urn, raw_log, tx_idx)
func (repository FrobRepository) Create(headerID int64, models []FrobModel) error {
tx, err := repository.db.Begin()
if err != nil {
return err
}
for _, model := range models {
_, err = tx.Exec(`INSERT INTO maker.frob (header_id, art, dart, dink, iart, ilk, ink, urn, raw_log, tx_idx)
VALUES($1, $2::NUMERIC, $3::NUMERIC, $4::NUMERIC, $5::NUMERIC, $6, $7::NUMERIC, $8, $9, $10)`,
headerID, model.Art, model.Dart, model.Dink, model.IArt, model.Ilk, model.Ink, model.Urn, model.Raw, model.TransactionIndex)
headerID, model.Art, model.Dart, model.Dink, model.IArt, model.Ilk, model.Ink, model.Urn, model.Raw, model.TransactionIndex)
if err != nil {
tx.Rollback()
return err
}
}
_, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, frob_checked)
VALUES ($1, $2)
ON CONFLICT (header_id) DO
UPDATE SET frob_checked = $2`, headerID, true)
if err != nil {
tx.Rollback()
return err
}
return tx.Commit()
}
func (repository FrobRepository) MarkHeaderChecked(headerID int64) error {
_, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, frob_checked)
VALUES ($1, $2)
ON CONFLICT (header_id) DO
UPDATE SET frob_checked = $2`, headerID, true)
return err
}
@@ -44,8 +71,8 @@ func (repository FrobRepository) MissingHeaders(startingBlockNumber, endingBlock
err := repository.db.Select(
&result,
`SELECT headers.id, headers.block_number FROM headers
LEFT JOIN maker.frob on headers.id = header_id
WHERE header_id ISNULL
LEFT JOIN checked_headers on headers.id = header_id
WHERE (header_id ISNULL OR frob_checked IS FALSE)
AND headers.block_number >= $1
AND headers.block_number <= $2
AND headers.eth_node_fingerprint = $3`,
pkg/transformers/frob/repository_test.go
+98 -23
View File
@@ -21,6 +21,7 @@ import (
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/frob"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
@@ -29,15 +30,24 @@ import (
var _ = Describe("Frob repository", func() {
Describe("Create", func() {
It("adds a frob", func() {
db := test_config.NewTestDB(core.Node{})
var (
db *postgres.DB
frobRepository frob.Repository
err error
headerID int64
)
BeforeEach(func() {
db = test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
frobRepository := frob.NewFrobRepository(db)
frobRepository = frob.NewFrobRepository(db)
})
err = frobRepository.Create(headerID, test_data.FrobModel)
It("adds a frob", func() {
err = frobRepository.Create(headerID, []frob.FrobModel{test_data.FrobModel})
Expect(err).NotTo(HaveOccurred())
var dbFrob frob.FrobModel
@@ -54,30 +64,28 @@ var _ = Describe("Frob repository", func() {
Expect(dbFrob.Raw).To(MatchJSON(test_data.FrobModel.Raw))
})
It("does not duplicate frob events", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
It("marks header as checked for logs", func() {
err = frobRepository.Create(headerID, []frob.FrobModel{test_data.FrobModel})
Expect(err).NotTo(HaveOccurred())
frobRepository := frob.NewFrobRepository(db)
err = frobRepository.Create(headerID, test_data.FrobModel)
var headerChecked bool
err = db.Get(&headerChecked, `SELECT frob_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue())
})
It("does not duplicate frob events", func() {
err = frobRepository.Create(headerID, []frob.FrobModel{test_data.FrobModel})
Expect(err).NotTo(HaveOccurred())
err = frobRepository.Create(headerID, test_data.FrobModel)
err = frobRepository.Create(headerID, []frob.FrobModel{test_data.FrobModel})
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
})
It("removes frob if corresponding header is deleted", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
frobRepository := frob.NewFrobRepository(db)
err = frobRepository.Create(headerID, test_data.FrobModel)
err = frobRepository.Create(headerID, []frob.FrobModel{test_data.FrobModel})
Expect(err).NotTo(HaveOccurred())
_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
@@ -90,8 +98,48 @@ var _ = Describe("Frob repository", func() {
})
})
Describe("MarkHeaderChecked", func() {
var (
db *postgres.DB
frobRepository frob.Repository
err error
headerID int64
)
BeforeEach(func() {
db = test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
frobRepository = frob.NewFrobRepository(db)
})
It("creates a row for a new headerID", func() {
err = frobRepository.MarkHeaderChecked(headerID)
Expect(err).NotTo(HaveOccurred())
var headerChecked bool
err = db.Get(&headerChecked, `SELECT frob_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue())
})
It("updates row when headerID already exists", func() {
_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID)
err = frobRepository.MarkHeaderChecked(headerID)
Expect(err).NotTo(HaveOccurred())
var headerChecked bool
err = db.Get(&headerChecked, `SELECT frob_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue())
})
})
Describe("MissingHeaders", func() {
It("returns headers with no associated frob event", func() {
It("returns headers that haven't been checked", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
@@ -106,7 +154,7 @@ var _ = Describe("Frob repository", func() {
Expect(err).NotTo(HaveOccurred())
}
frobRepository := frob.NewFrobRepository(db)
err := frobRepository.Create(headerIDs[1], test_data.FrobModel)
err := frobRepository.MarkHeaderChecked(headerIDs[1])
Expect(err).NotTo(HaveOccurred())
headers, err := frobRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
@@ -117,6 +165,33 @@ var _ = Describe("Frob repository", func() {
Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
})
It("only treats headers as checked if frob logs have been checked", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
startingBlockNumber := int64(1)
frobdBlockNumber := int64(2)
endingBlockNumber := int64(3)
blockNumbers := []int64{startingBlockNumber, frobdBlockNumber, endingBlockNumber, endingBlockNumber + 1}
var headerIDs []int64
for _, n := range blockNumbers {
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
headerIDs = append(headerIDs, headerID)
Expect(err).NotTo(HaveOccurred())
}
frobRepository := frob.NewFrobRepository(db)
_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
Expect(err).NotTo(HaveOccurred())
headers, err := frobRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
Expect(err).NotTo(HaveOccurred())
Expect(len(headers)).To(Equal(3))
Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(frobdBlockNumber)))
Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(frobdBlockNumber)))
Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(frobdBlockNumber)))
})
It("only returns headers associated with the current node", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
@@ -134,7 +209,7 @@ var _ = Describe("Frob repository", func() {
}
frobRepository := frob.NewFrobRepository(db)
frobRepositoryTwo := frob.NewFrobRepository(dbTwo)
err := frobRepository.Create(headerIDs[0], test_data.FrobModel)
err := frobRepository.MarkHeaderChecked(headerIDs[0])
Expect(err).NotTo(HaveOccurred())
nodeOneMissingHeaders, err := frobRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])
pkg/transformers/frob/transformer.go
+14 -10
View File
@@ -59,20 +59,24 @@ func (transformer FrobTransformer) Execute() error {
if err != nil {
return err
}
for _, log := range matchingLogs {
entity, err := transformer.Converter.ToEntity(log.Address.Hex(), FrobConfig.ContractAbi, log)
if err != nil {
return err
}
model, err := transformer.Converter.ToModel(entity)
if err != nil {
return err
}
err = transformer.Repository.Create(header.Id, model)
if len(matchingLogs) < 1 {
err = transformer.Repository.MarkHeaderChecked(header.Id)
if err != nil {
return err
}
}
entities, err := transformer.Converter.ToEntities(FrobConfig.ContractAbi, matchingLogs)
if err != nil {
return err
}
models, err := transformer.Converter.ToModels(entities)
if err != nil {
return err
}
err = transformer.Repository.Create(header.Id, models)
if err != nil {
return err
}
}
return nil
}
pkg/transformers/frob/transformer_test.go
+41 -6
View File
@@ -96,6 +96,42 @@ var _ = Describe("Frob transformer", func() {
Expect(err).To(MatchError(fakes.FakeError))
})
It("marks header checked if no logs returned", func() {
mockConverter := &frob_mocks.MockFrobConverter{}
mockRepository := &frob_mocks.MockFrobRepository{}
headerID := int64(123)
mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}})
mockFetcher := &mocks.MockLogFetcher{}
transformer := frob.FrobTransformer{
Converter: mockConverter,
Fetcher: mockFetcher,
Repository: mockRepository,
}
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
mockRepository.AssertMarkHeaderCheckedCalledWith(headerID)
})
It("returns error if marking header checked returns err", func() {
mockConverter := &frob_mocks.MockFrobConverter{}
mockRepository := &frob_mocks.MockFrobRepository{}
mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}})
mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError)
mockFetcher := &mocks.MockLogFetcher{}
transformer := frob.FrobTransformer{
Converter: mockConverter,
Fetcher: mockFetcher,
Repository: mockRepository,
}
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("converts matching logs to entity", func() {
converter := &frob_mocks.MockFrobConverter{}
fetcher := &mocks.MockLogFetcher{}
@@ -111,14 +147,13 @@ var _ = Describe("Frob transformer", func() {
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(converter.PassedContractAddress).To(Equal(test_data.EthFrobLog.Address.Hex()))
Expect(converter.PassedContractABI).To(Equal(frob.FrobConfig.ContractAbi))
Expect(converter.PassedLog).To(Equal(test_data.EthFrobLog))
Expect(converter.PassedLogs).To(Equal([]types.Log{test_data.EthFrobLog}))
})
It("returns error if converting to entity returns error", func() {
converter := &frob_mocks.MockFrobConverter{}
converter.SetToEntityError(fakes.FakeError)
converter.SetToEntitiesError(fakes.FakeError)
fetcher := &mocks.MockLogFetcher{}
fetcher.SetFetchedLogs([]types.Log{test_data.EthFrobLog})
repository := &frob_mocks.MockFrobRepository{}
@@ -150,12 +185,12 @@ var _ = Describe("Frob transformer", func() {
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(converter.PassedEntity).To(Equal(test_data.FrobEntity))
Expect(converter.PassedEntities).To(Equal([]frob.FrobEntity{test_data.FrobEntity}))
})
It("returns error if converting to model returns error", func() {
converter := &frob_mocks.MockFrobConverter{}
converter.SetToModelError(fakes.FakeError)
converter.SetToModelsError(fakes.FakeError)
fetcher := &mocks.MockLogFetcher{}
fetcher.SetFetchedLogs([]types.Log{test_data.EthFrobLog})
repository := &frob_mocks.MockFrobRepository{}
@@ -189,7 +224,7 @@ var _ = Describe("Frob transformer", func() {
Expect(err).NotTo(HaveOccurred())
Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id))
Expect(repository.PassedFrobModel).To(Equal(test_data.FrobModel))
Expect(repository.PassedModels).To(Equal([]frob.FrobModel{test_data.FrobModel}))
})
It("returns error if repository returns error for create", func() {