deep-stack/ipld-eth-server
4
0
Fork 0
forked from cerc-io/ipld-eth-server
Code Pull Requests Activity

Log checked headers for MCD events (#43)

Browse Source 
* Log checked headers for Bite events

* Log checked headers for Drip drip events

* Log checked headers for Drip file events

- mark headers checked for 3 implementations of file on the Drip contract

* Log checked headers for Frob frob events

* Log checked headers for Pit file events

- mark headers checked for 3 implementations of file on the Pit contract

* Log checked headers for Vat init events
This commit is contained in:
Rob Mulholand
2018-10-02 17:25:38 -05:00
committed by Elizabeth
parent 5a2d05b8d0
commit bf44ec614a
93 changed files with 2439 additions and 772 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/transformers/bite/converter.go
+56 -47
View File
@@ -19,66 +19,75 @@ package bite
import (
"encoding/json"
"github.com/ethereum/go-ethereum/accounts/abi/bind"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/geth"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
)
type Converter interface {
ToEntity(contractAddress string, contractAbi string, ethLog types.Log) (BiteEntity, error)
ToModel(flipKick BiteEntity) (BiteModel, error)
ToEntities(contractAbi string, ethLogs []types.Log) ([]BiteEntity, error)
ToModels(biteEntities []BiteEntity) ([]BiteModel, error)
}
type BiteConverter struct{}
func (BiteConverter) ToEntity(contractAddress string, contractAbi string, ethLog types.Log) (BiteEntity, error) {
entity := BiteEntity{}
address := common.HexToAddress(contractAddress)
abi, err := geth.ParseAbi(contractAbi)
if err != nil {
return entity, err
func (BiteConverter) ToEntities(contractAbi string, ethLogs []types.Log) ([]BiteEntity, error) {
var entities []BiteEntity
for _, ethLog := range ethLogs {
entity := BiteEntity{}
address := ethLog.Address
abi, err := geth.ParseAbi(contractAbi)
if err != nil {
return nil, err
}
contract := bind.NewBoundContract(address, abi, nil, nil, nil)
err = contract.UnpackLog(&entity, "Bite", ethLog)
if err != nil {
return nil, err
}
entity.Raw = ethLog
entity.TransactionIndex = ethLog.TxIndex
entities = append(entities, entity)
}
contract := bind.NewBoundContract(address, abi, nil, nil, nil)
err = contract.UnpackLog(&entity, "Bite", ethLog)
if err != nil {
return entity, err
}
entity.Raw = ethLog
entity.TransactionIndex = ethLog.TxIndex
return entity, nil
return entities, nil
}
func (converter BiteConverter) ToModel(entity BiteEntity) (BiteModel, error) {
id := entity.Id
ilk := entity.Ilk[:]
urn := entity.Urn[:]
ink := entity.Ink
art := entity.Art
iArt := entity.IArt
tab := entity.Tab
flip := entity.Flip
txIdx := entity.TransactionIndex
rawLogJson, err := json.Marshal(entity.Raw)
rawLogString := string(rawLogJson)
if err != nil {
return BiteModel{}, err
func (converter BiteConverter) ToModels(entities []BiteEntity) ([]BiteModel, error) {
var models []BiteModel
for _, entity := range entities {
id := entity.Id
ilk := entity.Ilk[:]
urn := entity.Urn[:]
ink := entity.Ink
art := entity.Art
iArt := entity.IArt
tab := entity.Tab
flip := entity.Flip
txIdx := entity.TransactionIndex
rawLogJson, err := json.Marshal(entity.Raw)
rawLogString := string(rawLogJson)
if err != nil {
return nil, err
}
model := BiteModel{
Id: shared.BigIntToString(id),
Ilk: ilk,
Urn: urn,
Ink: shared.BigIntToString(ink),
Art: shared.BigIntToString(art),
IArt: shared.BigIntToString(iArt),
Tab: shared.BigIntToString(tab),
Flip: shared.BigIntToString(flip),
TransactionIndex: txIdx,
Raw: rawLogString,
}
models = append(models, model)
}
return BiteModel{
Id: shared.BigIntToString(id),
Ilk: ilk,
Urn: urn,
Ink: shared.BigIntToString(ink),
Art: shared.BigIntToString(art),
IArt: shared.BigIntToString(iArt),
Tab: shared.BigIntToString(tab),
Flip: shared.BigIntToString(flip),
TransactionIndex: txIdx,
Raw: rawLogString,
}, nil
return models, nil
}
pkg/transformers/bite/converter_test.go
+10 -4
View File
@@ -34,9 +34,11 @@ var _ = Describe("Bite Converter", func() {
Describe("ToEntity", func() {
It("converts an eth log to a bite entity", func() {
entity, err := converter.ToEntity(shared.CatContractAddress, shared.CatABI, test_data.EthBiteLog)
entities, err := converter.ToEntities(shared.CatABI, []types.Log{test_data.EthBiteLog})
Expect(err).NotTo(HaveOccurred())
Expect(len(entities)).To(Equal(1))
entity := entities[0]
Expect(entity.Ilk).To(Equal(test_data.BiteEntity.Ilk))
Expect(entity.Urn).To(Equal(test_data.BiteEntity.Urn))
Expect(entity.Ink).To(Equal(test_data.BiteEntity.Ink))
@@ -49,7 +51,7 @@ var _ = Describe("Bite Converter", func() {
})
It("returns an error if converting log to entity fails", func() {
_, err := converter.ToEntity(shared.CatContractAddress, "error abi", test_data.EthBiteLog)
_, err := converter.ToEntities("error abi", []types.Log{test_data.EthBiteLog})
Expect(err).To(HaveOccurred())
})
@@ -63,9 +65,11 @@ var _ = Describe("Bite Converter", func() {
})
It("converts an Entity to a Model", func() {
model, err := converter.ToModel(test_data.BiteEntity)
models, err := converter.ToModels([]bite.BiteEntity{test_data.BiteEntity})
Expect(err).NotTo(HaveOccurred())
Expect(len(models)).To(Equal(1))
model := models[0]
Expect(model).To(Equal(test_data.BiteModel))
Expect(model.TransactionIndex).To(Equal(test_data.BiteModel.TransactionIndex))
})
@@ -85,9 +89,11 @@ var _ = Describe("Bite Converter", func() {
TransactionIndex: 0,
Raw: string(emptyLog),
}
model, err := converter.ToModel(emptyEntity)
models, err := converter.ToModels([]bite.BiteEntity{emptyEntity})
Expect(err).NotTo(HaveOccurred())
Expect(len(models)).To(Equal(1))
model := models[0]
Expect(model).To(Equal(expectedModel))
})
})
pkg/transformers/bite/repository.go
+35 -12
View File
@@ -20,7 +20,8 @@ import (
)
type Repository interface {
Create(headerID int64, model BiteModel) error
Create(headerID int64, models []BiteModel) error
MarkHeaderChecked(headerID int64) error
MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
}
@@ -32,26 +33,48 @@ func NewBiteRepository(db *postgres.DB) Repository {
return BiteRepository{db: db}
}
func (repository BiteRepository) Create(headerID int64, model BiteModel) error {
_, err := repository.db.Exec(
`INSERT into maker.bite (header_id, id, ilk, urn, ink, art, iart, tab, flip, tx_idx, raw_log)
VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`,
headerID, model.Id, model.Ilk, model.Urn, model.Ink, model.Art, model.IArt, model.Tab, model.Flip, model.TransactionIndex, model.Raw,
)
func (repository BiteRepository) Create(headerID int64, models []BiteModel) error {
tx, err := repository.db.Begin()
if err != nil {
return err
}
return nil
for _, model := range models {
_, err := tx.Exec(
`INSERT into maker.bite (header_id, id, ilk, urn, ink, art, iart, tab, flip, tx_idx, raw_log)
VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`,
headerID, model.Id, model.Ilk, model.Urn, model.Ink, model.Art, model.IArt, model.Tab, model.Flip, model.TransactionIndex, model.Raw,
)
if err != nil {
tx.Rollback()
return err
}
}
_, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, bite_checked)
VALUES ($1, $2)
ON CONFLICT (header_id) DO
UPDATE SET bite_checked = $2`, headerID, true)
if err != nil {
tx.Rollback()
return err
}
return tx.Commit()
}
func (repository BiteRepository) MarkHeaderChecked(headerID int64) error {
_, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, bite_checked)
VALUES ($1, $2)
ON CONFLICT (header_id) DO
UPDATE SET bite_checked = $2`, headerID, true)
return err
}
func (repository BiteRepository) MissingHeaders(startingBlockNumber int64, endingBlockNumber int64) ([]core.Header, error) {
var result []core.Header
err := repository.db.Select(
&result,
`SELECT headers.id, headers.block_number FROM headers
LEFT JOIN maker.bite on headers.id = header_id
WHERE header_id ISNULL
LEFT JOIN checked_headers on headers.id = header_id
WHERE (header_id ISNULL OR bite_checked IS FALSE)
AND headers.block_number >= $1
AND headers.block_number <= $2
AND headers.eth_node_fingerprint = $3`,
pkg/transformers/bite/repository_test.go
+67 -25
View File
@@ -20,6 +20,7 @@ import (
"database/sql"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/bite"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
@@ -28,16 +29,24 @@ import (
var _ = Describe("Bite repository", func() {
Describe("Create", func() {
It("persists a bite record", func() {
node := core.Node{}
db := test_config.NewTestDB(node)
var (
biteRepository bite.Repository
db *postgres.DB
err error
headerID int64
)
BeforeEach(func() {
db = test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
biteRepository := bite.NewBiteRepository(db)
biteRepository = bite.NewBiteRepository(db)
})
err = biteRepository.Create(headerID, test_data.BiteModel)
It("persists a bite record", func() {
err = biteRepository.Create(headerID, []bite.BiteModel{test_data.BiteModel})
Expect(err).NotTo(HaveOccurred())
var dbBite bite.BiteModel
@@ -53,15 +62,18 @@ var _ = Describe("Bite repository", func() {
Expect(dbBite.Raw).To(MatchJSON(test_data.BiteModel.Raw))
})
It("does not duplicate bite events", func() {
node := core.Node{}
db := test_config.NewTestDB(node)
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
It("marks header as checked for logs", func() {
err = biteRepository.Create(headerID, []bite.BiteModel{test_data.BiteModel})
Expect(err).NotTo(HaveOccurred())
biteRepository := bite.NewBiteRepository(db)
err = biteRepository.Create(headerID, test_data.BiteModel)
var headerChecked bool
err = db.Get(&headerChecked, `SELECT bite_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue())
})
It("does not duplicate bite events", func() {
err = biteRepository.Create(headerID, []bite.BiteModel{test_data.BiteModel})
Expect(err).NotTo(HaveOccurred())
var anotherBiteModel = bite.BiteModel{
@@ -77,21 +89,14 @@ var _ = Describe("Bite repository", func() {
Raw: test_data.BiteModel.Raw,
}
err = biteRepository.Create(headerID, anotherBiteModel)
err = biteRepository.Create(headerID, []bite.BiteModel{anotherBiteModel})
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
})
It("removes bite if corresponding header is deleted", func() {
node := core.Node{}
db := test_config.NewTestDB(node)
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
biteRepository := bite.NewBiteRepository(db)
err = biteRepository.Create(headerID, test_data.BiteModel)
err = biteRepository.Create(headerID, []bite.BiteModel{test_data.BiteModel})
Expect(err).NotTo(HaveOccurred())
_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
@@ -104,8 +109,45 @@ var _ = Describe("Bite repository", func() {
})
})
Describe("MarkHeaderChecked", func() {
It("creates a row for a new headerID", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
biteRepository := bite.NewBiteRepository(db)
err = biteRepository.MarkHeaderChecked(headerID)
Expect(err).NotTo(HaveOccurred())
var headerChecked bool
err = db.Get(&headerChecked, `SELECT bite_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue())
})
It("updates row when headerID already exists", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
biteRepository := bite.NewBiteRepository(db)
_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID)
err = biteRepository.MarkHeaderChecked(headerID)
Expect(err).NotTo(HaveOccurred())
var headerChecked bool
err = db.Get(&headerChecked, `SELECT bite_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue())
})
})
Describe("MissingHeaders", func() {
It("returns headers with no associated bite event", func() {
It("returns headers that haven't been checked", func() {
node := core.Node{}
db := test_config.NewTestDB(node)
test_config.CleanTestDB(db)
@@ -121,7 +163,7 @@ var _ = Describe("Bite repository", func() {
Expect(err).NotTo(HaveOccurred())
}
biteRepository := bite.NewBiteRepository(db)
err := biteRepository.Create(headerIDs[1], test_data.BiteModel)
err := biteRepository.MarkHeaderChecked(headerIDs[1])
Expect(err).NotTo(HaveOccurred())
headers, err := biteRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
pkg/transformers/bite/transformer.go
+16 -10
View File
@@ -67,20 +67,26 @@ func (b BiteTransformer) Execute() error {
log.Println("Error fetching matching logs:", err)
return err
}
for _, ethLog := range ethLogs {
entity, err := b.Converter.ToEntity(ethLog.Address.Hex(), config.ContractAbi, ethLog)
model, err := b.Converter.ToModel(entity)
if len(ethLogs) < 1 {
err = b.Repository.MarkHeaderChecked(header.Id)
if err != nil {
log.Println("Error converting logs:", err)
return err
}
}
entities, err := b.Converter.ToEntities(config.ContractAbi, ethLogs)
if err != nil {
return err
}
model, err := b.Converter.ToModels(entities)
if err != nil {
log.Println("Error converting logs:", err)
return err
}
err = b.Repository.Create(header.Id, model)
if err != nil {
log.Println("Error persisting bite record:", err)
return err
}
err = b.Repository.Create(header.Id, model)
if err != nil {
log.Println("Error persisting bite record:", err)
return err
}
}
pkg/transformers/bite/transfromer_test.go
+24 -3
View File
@@ -17,10 +17,13 @@
package bite_test
import (
"math/rand"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/transformers/bite"
@@ -28,7 +31,6 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks"
bite_mocks "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks/bite"
"math/rand"
)
var _ = Describe("Bite Transformer", func() {
@@ -89,13 +91,32 @@ var _ = Describe("Bite Transformer", func() {
Expect(err).To(MatchError(fakes.FakeError))
})
It("marks header checked if no logs returned", func() {
headerID := int64(123)
repository.SetMissingHeaders([]core.Header{{Id: headerID}})
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
repository.AssertMarkHeaderCheckedCalledWith(headerID)
})
It("returns error if marking header checked returns err", func() {
repository.SetMissingHeaders([]core.Header{{Id: int64(123)}})
repository.SetMarkHeaderCheckedErr(fakes.FakeError)
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("converts an eth log to an Entity", func() {
repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}})
fetcher.SetFetchedLogs([]types.Log{test_data.EthBiteLog})
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(converter.ConverterContract).To(Equal(test_data.EthBiteLog.Address.Hex()))
Expect(converter.ConverterAbi).To(Equal(bite.BiteConfig.ContractAbi))
Expect(converter.LogsToConvert).To(Equal([]types.Log{test_data.EthBiteLog}))
})
@@ -121,7 +142,7 @@ var _ = Describe("Bite Transformer", func() {
Expect(err).NotTo(HaveOccurred())
Expect(repository.PassedHeaderID).To(Equal(headerId))
Expect(repository.PassedBiteModel).To(Equal(test_data.BiteModel))
Expect(repository.PassedBiteModels).To(Equal([]bite.BiteModel{test_data.BiteModel}))
})
It("returns error if persisting bite record fails", func() {