Log checked headers for MCD events (#43)

* Log checked headers for Bite events

* Log checked headers for Drip drip events

* Log checked headers for Drip file events

- mark headers checked for 3 implementations of file on the Drip contract

* Log checked headers for Frob frob events

* Log checked headers for Pit file events

- mark headers checked for 3 implementations of file on the Pit contract

* Log checked headers for Vat init events
This commit is contained in:
Rob Mulholand 2018-10-02 17:25:38 -05:00 committed by Elizabeth
parent 5a2d05b8d0
commit bf44ec614a
93 changed files with 2439 additions and 772 deletions

View File

@ -0,0 +1,2 @@
ALTER TABLE public.checked_headers
DROP COLUMN bite_checked;

View File

@ -0,0 +1,2 @@
ALTER TABLE public.checked_headers
ADD COLUMN bite_checked BOOLEAN NOT NULL DEFAULT FALSE;

View File

@ -0,0 +1,2 @@
ALTER TABLE public.checked_headers
DROP COLUMN drip_drip_checked;

View File

@ -0,0 +1,2 @@
ALTER TABLE public.checked_headers
ADD COLUMN drip_drip_checked BOOLEAN NOT NULL DEFAULT FALSE;

View File

@ -0,0 +1,6 @@
ALTER TABLE public.checked_headers
DROP COLUMN drip_file_ilk_checked;
ALTER TABLE public.checked_headers
DROP COLUMN drip_file_repo_checked;
ALTER TABLE public.checked_headers
DROP COLUMN drip_file_vow_checked;

View File

@ -0,0 +1,6 @@
ALTER TABLE public.checked_headers
ADD COLUMN drip_file_ilk_checked BOOLEAN NOT NULL DEFAULT FALSE;
ALTER TABLE public.checked_headers
ADD COLUMN drip_file_repo_checked BOOLEAN NOT NULL DEFAULT FALSE;
ALTER TABLE public.checked_headers
ADD COLUMN drip_file_vow_checked BOOLEAN NOT NULL DEFAULT FALSE;

View File

@ -0,0 +1,2 @@
ALTER TABLE public.checked_headers
DROP COLUMN frob_checked;

View File

@ -0,0 +1,2 @@
ALTER TABLE public.checked_headers
ADD COLUMN frob_checked BOOLEAN NOT NULL DEFAULT FALSE;

View File

@ -0,0 +1,6 @@
ALTER TABLE public.checked_headers
DROP COLUMN pit_file_debt_ceiling_checked;
ALTER TABLE public.checked_headers
DROP COLUMN pit_file_ilk_checked;
ALTER TABLE public.checked_headers
DROP COLUMN pit_file_stability_fee_checked;

View File

@ -0,0 +1,6 @@
ALTER TABLE public.checked_headers
ADD COLUMN pit_file_debt_ceiling_checked BOOLEAN NOT NULL DEFAULT FALSE;
ALTER TABLE public.checked_headers
ADD COLUMN pit_file_ilk_checked BOOLEAN NOT NULL DEFAULT FALSE;
ALTER TABLE public.checked_headers
ADD COLUMN pit_file_stability_fee_checked BOOLEAN NOT NULL DEFAULT FALSE;

View File

@ -0,0 +1,2 @@
ALTER TABLE public.checked_headers
DROP COLUMN vat_init_checked;

View File

@ -0,0 +1,2 @@
ALTER TABLE public.checked_headers
ADD COLUMN vat_init_checked BOOLEAN NOT NULL DEFAULT FALSE;

View File

@ -820,7 +820,17 @@ CREATE TABLE public.checked_headers (
tend_checked boolean DEFAULT false NOT NULL,
cat_file_chop_lump_checked boolean DEFAULT false NOT NULL,
cat_file_flip_checked boolean DEFAULT false NOT NULL,
cat_file_pit_vow_checked boolean DEFAULT false NOT NULL
cat_file_pit_vow_checked boolean DEFAULT false NOT NULL,
bite_checked boolean DEFAULT false NOT NULL,
drip_drip_checked boolean DEFAULT false NOT NULL,
drip_file_ilk_checked boolean DEFAULT false NOT NULL,
drip_file_repo_checked boolean DEFAULT false NOT NULL,
drip_file_vow_checked boolean DEFAULT false NOT NULL,
frob_checked boolean DEFAULT false NOT NULL,
pit_file_debt_ceiling_checked boolean DEFAULT false NOT NULL,
pit_file_ilk_checked boolean DEFAULT false NOT NULL,
pit_file_stability_fee_checked boolean DEFAULT false NOT NULL,
vat_init_checked boolean DEFAULT false NOT NULL
);

View File

@ -19,66 +19,75 @@ package bite
import (
"encoding/json"
"github.com/ethereum/go-ethereum/accounts/abi/bind"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/geth"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
)
type Converter interface {
ToEntity(contractAddress string, contractAbi string, ethLog types.Log) (BiteEntity, error)
ToModel(flipKick BiteEntity) (BiteModel, error)
ToEntities(contractAbi string, ethLogs []types.Log) ([]BiteEntity, error)
ToModels(biteEntities []BiteEntity) ([]BiteModel, error)
}
type BiteConverter struct{}
func (BiteConverter) ToEntity(contractAddress string, contractAbi string, ethLog types.Log) (BiteEntity, error) {
entity := BiteEntity{}
address := common.HexToAddress(contractAddress)
abi, err := geth.ParseAbi(contractAbi)
if err != nil {
return entity, err
func (BiteConverter) ToEntities(contractAbi string, ethLogs []types.Log) ([]BiteEntity, error) {
var entities []BiteEntity
for _, ethLog := range ethLogs {
entity := BiteEntity{}
address := ethLog.Address
abi, err := geth.ParseAbi(contractAbi)
if err != nil {
return nil, err
}
contract := bind.NewBoundContract(address, abi, nil, nil, nil)
err = contract.UnpackLog(&entity, "Bite", ethLog)
if err != nil {
return nil, err
}
entity.Raw = ethLog
entity.TransactionIndex = ethLog.TxIndex
entities = append(entities, entity)
}
contract := bind.NewBoundContract(address, abi, nil, nil, nil)
err = contract.UnpackLog(&entity, "Bite", ethLog)
if err != nil {
return entity, err
}
entity.Raw = ethLog
entity.TransactionIndex = ethLog.TxIndex
return entity, nil
return entities, nil
}
func (converter BiteConverter) ToModel(entity BiteEntity) (BiteModel, error) {
id := entity.Id
ilk := entity.Ilk[:]
urn := entity.Urn[:]
ink := entity.Ink
art := entity.Art
iArt := entity.IArt
tab := entity.Tab
flip := entity.Flip
txIdx := entity.TransactionIndex
rawLogJson, err := json.Marshal(entity.Raw)
rawLogString := string(rawLogJson)
if err != nil {
return BiteModel{}, err
func (converter BiteConverter) ToModels(entities []BiteEntity) ([]BiteModel, error) {
var models []BiteModel
for _, entity := range entities {
id := entity.Id
ilk := entity.Ilk[:]
urn := entity.Urn[:]
ink := entity.Ink
art := entity.Art
iArt := entity.IArt
tab := entity.Tab
flip := entity.Flip
txIdx := entity.TransactionIndex
rawLogJson, err := json.Marshal(entity.Raw)
rawLogString := string(rawLogJson)
if err != nil {
return nil, err
}
model := BiteModel{
Id: shared.BigIntToString(id),
Ilk: ilk,
Urn: urn,
Ink: shared.BigIntToString(ink),
Art: shared.BigIntToString(art),
IArt: shared.BigIntToString(iArt),
Tab: shared.BigIntToString(tab),
Flip: shared.BigIntToString(flip),
TransactionIndex: txIdx,
Raw: rawLogString,
}
models = append(models, model)
}
return BiteModel{
Id: shared.BigIntToString(id),
Ilk: ilk,
Urn: urn,
Ink: shared.BigIntToString(ink),
Art: shared.BigIntToString(art),
IArt: shared.BigIntToString(iArt),
Tab: shared.BigIntToString(tab),
Flip: shared.BigIntToString(flip),
TransactionIndex: txIdx,
Raw: rawLogString,
}, nil
return models, nil
}

View File

@ -34,9 +34,11 @@ var _ = Describe("Bite Converter", func() {
Describe("ToEntity", func() {
It("converts an eth log to a bite entity", func() {
entity, err := converter.ToEntity(shared.CatContractAddress, shared.CatABI, test_data.EthBiteLog)
entities, err := converter.ToEntities(shared.CatABI, []types.Log{test_data.EthBiteLog})
Expect(err).NotTo(HaveOccurred())
Expect(len(entities)).To(Equal(1))
entity := entities[0]
Expect(entity.Ilk).To(Equal(test_data.BiteEntity.Ilk))
Expect(entity.Urn).To(Equal(test_data.BiteEntity.Urn))
Expect(entity.Ink).To(Equal(test_data.BiteEntity.Ink))
@ -49,7 +51,7 @@ var _ = Describe("Bite Converter", func() {
})
It("returns an error if converting log to entity fails", func() {
_, err := converter.ToEntity(shared.CatContractAddress, "error abi", test_data.EthBiteLog)
_, err := converter.ToEntities("error abi", []types.Log{test_data.EthBiteLog})
Expect(err).To(HaveOccurred())
})
@ -63,9 +65,11 @@ var _ = Describe("Bite Converter", func() {
})
It("converts an Entity to a Model", func() {
model, err := converter.ToModel(test_data.BiteEntity)
models, err := converter.ToModels([]bite.BiteEntity{test_data.BiteEntity})
Expect(err).NotTo(HaveOccurred())
Expect(len(models)).To(Equal(1))
model := models[0]
Expect(model).To(Equal(test_data.BiteModel))
Expect(model.TransactionIndex).To(Equal(test_data.BiteModel.TransactionIndex))
})
@ -85,9 +89,11 @@ var _ = Describe("Bite Converter", func() {
TransactionIndex: 0,
Raw: string(emptyLog),
}
model, err := converter.ToModel(emptyEntity)
models, err := converter.ToModels([]bite.BiteEntity{emptyEntity})
Expect(err).NotTo(HaveOccurred())
Expect(len(models)).To(Equal(1))
model := models[0]
Expect(model).To(Equal(expectedModel))
})
})

View File

@ -20,7 +20,8 @@ import (
)
type Repository interface {
Create(headerID int64, model BiteModel) error
Create(headerID int64, models []BiteModel) error
MarkHeaderChecked(headerID int64) error
MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
}
@ -32,26 +33,48 @@ func NewBiteRepository(db *postgres.DB) Repository {
return BiteRepository{db: db}
}
func (repository BiteRepository) Create(headerID int64, model BiteModel) error {
_, err := repository.db.Exec(
`INSERT into maker.bite (header_id, id, ilk, urn, ink, art, iart, tab, flip, tx_idx, raw_log)
VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`,
headerID, model.Id, model.Ilk, model.Urn, model.Ink, model.Art, model.IArt, model.Tab, model.Flip, model.TransactionIndex, model.Raw,
)
func (repository BiteRepository) Create(headerID int64, models []BiteModel) error {
tx, err := repository.db.Begin()
if err != nil {
return err
}
return nil
for _, model := range models {
_, err := tx.Exec(
`INSERT into maker.bite (header_id, id, ilk, urn, ink, art, iart, tab, flip, tx_idx, raw_log)
VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`,
headerID, model.Id, model.Ilk, model.Urn, model.Ink, model.Art, model.IArt, model.Tab, model.Flip, model.TransactionIndex, model.Raw,
)
if err != nil {
tx.Rollback()
return err
}
}
_, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, bite_checked)
VALUES ($1, $2)
ON CONFLICT (header_id) DO
UPDATE SET bite_checked = $2`, headerID, true)
if err != nil {
tx.Rollback()
return err
}
return tx.Commit()
}
func (repository BiteRepository) MarkHeaderChecked(headerID int64) error {
_, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, bite_checked)
VALUES ($1, $2)
ON CONFLICT (header_id) DO
UPDATE SET bite_checked = $2`, headerID, true)
return err
}
func (repository BiteRepository) MissingHeaders(startingBlockNumber int64, endingBlockNumber int64) ([]core.Header, error) {
var result []core.Header
err := repository.db.Select(
&result,
`SELECT headers.id, headers.block_number FROM headers
LEFT JOIN maker.bite on headers.id = header_id
WHERE header_id ISNULL
LEFT JOIN checked_headers on headers.id = header_id
WHERE (header_id ISNULL OR bite_checked IS FALSE)
AND headers.block_number >= $1
AND headers.block_number <= $2
AND headers.eth_node_fingerprint = $3`,

View File

@ -20,6 +20,7 @@ import (
"database/sql"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/bite"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
@ -28,16 +29,24 @@ import (
var _ = Describe("Bite repository", func() {
Describe("Create", func() {
It("persists a bite record", func() {
node := core.Node{}
db := test_config.NewTestDB(node)
var (
biteRepository bite.Repository
db *postgres.DB
err error
headerID int64
)
BeforeEach(func() {
db = test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
biteRepository := bite.NewBiteRepository(db)
biteRepository = bite.NewBiteRepository(db)
})
err = biteRepository.Create(headerID, test_data.BiteModel)
It("persists a bite record", func() {
err = biteRepository.Create(headerID, []bite.BiteModel{test_data.BiteModel})
Expect(err).NotTo(HaveOccurred())
var dbBite bite.BiteModel
@ -53,15 +62,18 @@ var _ = Describe("Bite repository", func() {
Expect(dbBite.Raw).To(MatchJSON(test_data.BiteModel.Raw))
})
It("does not duplicate bite events", func() {
node := core.Node{}
db := test_config.NewTestDB(node)
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
It("marks header as checked for logs", func() {
err = biteRepository.Create(headerID, []bite.BiteModel{test_data.BiteModel})
Expect(err).NotTo(HaveOccurred())
biteRepository := bite.NewBiteRepository(db)
err = biteRepository.Create(headerID, test_data.BiteModel)
var headerChecked bool
err = db.Get(&headerChecked, `SELECT bite_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue())
})
It("does not duplicate bite events", func() {
err = biteRepository.Create(headerID, []bite.BiteModel{test_data.BiteModel})
Expect(err).NotTo(HaveOccurred())
var anotherBiteModel = bite.BiteModel{
@ -77,21 +89,14 @@ var _ = Describe("Bite repository", func() {
Raw: test_data.BiteModel.Raw,
}
err = biteRepository.Create(headerID, anotherBiteModel)
err = biteRepository.Create(headerID, []bite.BiteModel{anotherBiteModel})
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
})
It("removes bite if corresponding header is deleted", func() {
node := core.Node{}
db := test_config.NewTestDB(node)
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
biteRepository := bite.NewBiteRepository(db)
err = biteRepository.Create(headerID, test_data.BiteModel)
err = biteRepository.Create(headerID, []bite.BiteModel{test_data.BiteModel})
Expect(err).NotTo(HaveOccurred())
_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
@ -104,8 +109,45 @@ var _ = Describe("Bite repository", func() {
})
})
Describe("MarkHeaderChecked", func() {
It("creates a row for a new headerID", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
biteRepository := bite.NewBiteRepository(db)
err = biteRepository.MarkHeaderChecked(headerID)
Expect(err).NotTo(HaveOccurred())
var headerChecked bool
err = db.Get(&headerChecked, `SELECT bite_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue())
})
It("updates row when headerID already exists", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
biteRepository := bite.NewBiteRepository(db)
_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID)
err = biteRepository.MarkHeaderChecked(headerID)
Expect(err).NotTo(HaveOccurred())
var headerChecked bool
err = db.Get(&headerChecked, `SELECT bite_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue())
})
})
Describe("MissingHeaders", func() {
It("returns headers with no associated bite event", func() {
It("returns headers that haven't been checked", func() {
node := core.Node{}
db := test_config.NewTestDB(node)
test_config.CleanTestDB(db)
@ -121,7 +163,7 @@ var _ = Describe("Bite repository", func() {
Expect(err).NotTo(HaveOccurred())
}
biteRepository := bite.NewBiteRepository(db)
err := biteRepository.Create(headerIDs[1], test_data.BiteModel)
err := biteRepository.MarkHeaderChecked(headerIDs[1])
Expect(err).NotTo(HaveOccurred())
headers, err := biteRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)

View File

@ -67,20 +67,26 @@ func (b BiteTransformer) Execute() error {
log.Println("Error fetching matching logs:", err)
return err
}
for _, ethLog := range ethLogs {
entity, err := b.Converter.ToEntity(ethLog.Address.Hex(), config.ContractAbi, ethLog)
model, err := b.Converter.ToModel(entity)
if len(ethLogs) < 1 {
err = b.Repository.MarkHeaderChecked(header.Id)
if err != nil {
log.Println("Error converting logs:", err)
return err
}
}
entities, err := b.Converter.ToEntities(config.ContractAbi, ethLogs)
if err != nil {
return err
}
model, err := b.Converter.ToModels(entities)
if err != nil {
log.Println("Error converting logs:", err)
return err
}
err = b.Repository.Create(header.Id, model)
if err != nil {
log.Println("Error persisting bite record:", err)
return err
}
err = b.Repository.Create(header.Id, model)
if err != nil {
log.Println("Error persisting bite record:", err)
return err
}
}

View File

@ -17,10 +17,13 @@
package bite_test
import (
"math/rand"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/transformers/bite"
@ -28,7 +31,6 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks"
bite_mocks "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks/bite"
"math/rand"
)
var _ = Describe("Bite Transformer", func() {
@ -89,13 +91,32 @@ var _ = Describe("Bite Transformer", func() {
Expect(err).To(MatchError(fakes.FakeError))
})
It("marks header checked if no logs returned", func() {
headerID := int64(123)
repository.SetMissingHeaders([]core.Header{{Id: headerID}})
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
repository.AssertMarkHeaderCheckedCalledWith(headerID)
})
It("returns error if marking header checked returns err", func() {
repository.SetMissingHeaders([]core.Header{{Id: int64(123)}})
repository.SetMarkHeaderCheckedErr(fakes.FakeError)
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("converts an eth log to an Entity", func() {
repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}})
fetcher.SetFetchedLogs([]types.Log{test_data.EthBiteLog})
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(converter.ConverterContract).To(Equal(test_data.EthBiteLog.Address.Hex()))
Expect(converter.ConverterAbi).To(Equal(bite.BiteConfig.ContractAbi))
Expect(converter.LogsToConvert).To(Equal([]types.Log{test_data.EthBiteLog}))
})
@ -121,7 +142,7 @@ var _ = Describe("Bite Transformer", func() {
Expect(err).NotTo(HaveOccurred())
Expect(repository.PassedHeaderID).To(Equal(headerId))
Expect(repository.PassedBiteModel).To(Equal(test_data.BiteModel))
Expect(repository.PassedBiteModels).To(Equal([]bite.BiteModel{test_data.BiteModel}))
})
It("returns error if persisting bite record fails", func() {

View File

@ -22,23 +22,31 @@ import (
)
type Converter interface {
ToModel(ethLog types.Log) (DripDripModel, error)
ToModels(ethLogs []types.Log) ([]DripDripModel, error)
}
type DripDripConverter struct{}
func (DripDripConverter) ToModel(ethLog types.Log) (DripDripModel, error) {
err := verifyLog(ethLog)
if err != nil {
return DripDripModel{}, err
func (DripDripConverter) ToModels(ethLogs []types.Log) ([]DripDripModel, error) {
var models []DripDripModel
for _, ethLog := range ethLogs {
err := verifyLog(ethLog)
if err != nil {
return nil, err
}
ilk := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
raw, err := json.Marshal(ethLog)
if err != nil {
return nil, err
}
model := DripDripModel{
Ilk: ilk,
TransactionIndex: ethLog.TxIndex,
Raw: raw,
}
models = append(models, model)
}
ilk := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
raw, err := json.Marshal(ethLog)
return DripDripModel{
Ilk: ilk,
TransactionIndex: ethLog.TxIndex,
Raw: raw,
}, err
return models, nil
}
func verifyLog(log types.Log) error {

View File

@ -28,7 +28,7 @@ var _ = Describe("Drip drip converter", func() {
converter := drip_drip.DripDripConverter{}
badLog := types.Log{}
_, err := converter.ToModel(badLog)
_, err := converter.ToModels([]types.Log{badLog})
Expect(err).To(HaveOccurred())
})
@ -36,9 +36,9 @@ var _ = Describe("Drip drip converter", func() {
It("converts a log to an model", func() {
converter := drip_drip.DripDripConverter{}
model, err := converter.ToModel(test_data.EthDripDripLog)
model, err := converter.ToModels([]types.Log{test_data.EthDripDripLog})
Expect(err).NotTo(HaveOccurred())
Expect(model).To(Equal(test_data.DripDripModel))
Expect(model).To(Equal([]drip_drip.DripDripModel{test_data.DripDripModel}))
})
})

View File

@ -20,7 +20,8 @@ import (
)
type Repository interface {
Create(headerID int64, model DripDripModel) error
Create(headerID int64, models []DripDripModel) error
MarkHeaderChecked(headerID int64) error
MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
}
@ -32,12 +33,38 @@ func NewDripDripRepository(db *postgres.DB) DripDripRepository {
return DripDripRepository{db: db}
}
func (repository DripDripRepository) Create(headerID int64, model DripDripModel) error {
_, err := repository.db.Exec(
`INSERT into maker.drip_drip (header_id, ilk, tx_idx, raw_log)
func (repository DripDripRepository) Create(headerID int64, models []DripDripModel) error {
tx, err := repository.db.Begin()
if err != nil {
return err
}
for _, model := range models {
_, err = tx.Exec(
`INSERT into maker.drip_drip (header_id, ilk, tx_idx, raw_log)
VALUES($1, $2, $3, $4)`,
headerID, model.Ilk, model.TransactionIndex, model.Raw,
)
headerID, model.Ilk, model.TransactionIndex, model.Raw,
)
if err != nil {
tx.Rollback()
return err
}
}
_, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, drip_drip_checked)
VALUES ($1, $2)
ON CONFLICT (header_id) DO
UPDATE SET drip_drip_checked = $2`, headerID, true)
if err != nil {
tx.Rollback()
return err
}
return tx.Commit()
}
func (repository DripDripRepository) MarkHeaderChecked(headerID int64) error {
_, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, drip_drip_checked)
VALUES ($1, $2)
ON CONFLICT (header_id) DO
UPDATE SET drip_drip_checked = $2`, headerID, true)
return err
}
@ -46,8 +73,8 @@ func (repository DripDripRepository) MissingHeaders(startingBlockNumber, endingB
err := repository.db.Select(
&result,
`SELECT headers.id, headers.block_number FROM headers
LEFT JOIN maker.drip_drip on headers.id = header_id
WHERE header_id ISNULL
LEFT JOIN checked_headers on headers.id = header_id
WHERE (header_id ISNULL OR drip_drip_checked IS FALSE)
AND headers.block_number >= $1
AND headers.block_number <= $2
AND headers.eth_node_fingerprint = $3`,

View File

@ -21,6 +21,7 @@ import (
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/drip_drip"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
@ -29,15 +30,24 @@ import (
var _ = Describe("Drip drip repository", func() {
Describe("Create", func() {
It("adds a drip drip event", func() {
db := test_config.NewTestDB(core.Node{})
var (
db *postgres.DB
dripDripRepository drip_drip.Repository
err error
headerID int64
)
BeforeEach(func() {
db = test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
dripDripRepository := drip_drip.NewDripDripRepository(db)
dripDripRepository = drip_drip.NewDripDripRepository(db)
})
err = dripDripRepository.Create(headerID, test_data.DripDripModel)
It("adds a drip drip event", func() {
err = dripDripRepository.Create(headerID, []drip_drip.DripDripModel{test_data.DripDripModel})
Expect(err).NotTo(HaveOccurred())
var dbPitFile drip_drip.DripDripModel
@ -48,30 +58,28 @@ var _ = Describe("Drip drip repository", func() {
Expect(dbPitFile.Raw).To(MatchJSON(test_data.DripDripModel.Raw))
})
It("does not duplicate drip drip events", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
It("marks header as checked for logs", func() {
err = dripDripRepository.Create(headerID, []drip_drip.DripDripModel{test_data.DripDripModel})
Expect(err).NotTo(HaveOccurred())
dripDripRepository := drip_drip.NewDripDripRepository(db)
err = dripDripRepository.Create(headerID, test_data.DripDripModel)
var headerChecked bool
err = db.Get(&headerChecked, `SELECT drip_drip_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue())
})
It("does not duplicate drip drip events", func() {
err = dripDripRepository.Create(headerID, []drip_drip.DripDripModel{test_data.DripDripModel})
Expect(err).NotTo(HaveOccurred())
err = dripDripRepository.Create(headerID, test_data.DripDripModel)
err = dripDripRepository.Create(headerID, []drip_drip.DripDripModel{test_data.DripDripModel})
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
})
It("removes drip drip if corresponding header is deleted", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
dripDripRepository := drip_drip.NewDripDripRepository(db)
err = dripDripRepository.Create(headerID, test_data.DripDripModel)
err = dripDripRepository.Create(headerID, []drip_drip.DripDripModel{test_data.DripDripModel})
Expect(err).NotTo(HaveOccurred())
_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
@ -84,8 +92,48 @@ var _ = Describe("Drip drip repository", func() {
})
})
Describe("MarkHeaderChecked", func() {
var (
db *postgres.DB
dripDripRepository drip_drip.Repository
err error
headerID int64
)
BeforeEach(func() {
db = test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
dripDripRepository = drip_drip.NewDripDripRepository(db)
})
It("creates a row for a new headerID", func() {
err = dripDripRepository.MarkHeaderChecked(headerID)
Expect(err).NotTo(HaveOccurred())
var headerChecked bool
err = db.Get(&headerChecked, `SELECT drip_drip_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue())
})
It("updates row when headerID already exists", func() {
_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID)
err = dripDripRepository.MarkHeaderChecked(headerID)
Expect(err).NotTo(HaveOccurred())
var headerChecked bool
err = db.Get(&headerChecked, `SELECT drip_drip_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue())
})
})
Describe("MissingHeaders", func() {
It("returns headers with no associated drip drip event", func() {
It("returns headers that haven't been checked", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
@ -100,7 +148,7 @@ var _ = Describe("Drip drip repository", func() {
Expect(err).NotTo(HaveOccurred())
}
dripDripRepository := drip_drip.NewDripDripRepository(db)
err := dripDripRepository.Create(headerIDs[1], test_data.DripDripModel)
err := dripDripRepository.MarkHeaderChecked(headerIDs[1])
Expect(err).NotTo(HaveOccurred())
headers, err := dripDripRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
@ -111,6 +159,33 @@ var _ = Describe("Drip drip repository", func() {
Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
})
It("only treats headers as checked if drip drip logs have been checked", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
startingBlockNumber := int64(1)
dripDripBlockNumber := int64(2)
endingBlockNumber := int64(3)
blockNumbers := []int64{startingBlockNumber, dripDripBlockNumber, endingBlockNumber, endingBlockNumber + 1}
var headerIDs []int64
for _, n := range blockNumbers {
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
headerIDs = append(headerIDs, headerID)
Expect(err).NotTo(HaveOccurred())
}
dripDripRepository := drip_drip.NewDripDripRepository(db)
_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
Expect(err).NotTo(HaveOccurred())
headers, err := dripDripRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
Expect(err).NotTo(HaveOccurred())
Expect(len(headers)).To(Equal(3))
Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripDripBlockNumber)))
Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripDripBlockNumber)))
Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripDripBlockNumber)))
})
It("only returns headers associated with the current node", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
@ -128,7 +203,7 @@ var _ = Describe("Drip drip repository", func() {
}
dripDripRepository := drip_drip.NewDripDripRepository(db)
dripDripRepositoryTwo := drip_drip.NewDripDripRepository(dbTwo)
err := dripDripRepository.Create(headerIDs[0], test_data.DripDripModel)
err := dripDripRepository.MarkHeaderChecked(headerIDs[0])
Expect(err).NotTo(HaveOccurred())
nodeOneMissingHeaders, err := dripDripRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])

View File

@ -59,16 +59,20 @@ func (transformer DripDripTransformer) Execute() error {
if err != nil {
return err
}
for _, log := range matchingLogs {
model, err := transformer.Converter.ToModel(log)
if err != nil {
return err
}
err = transformer.Repository.Create(header.Id, model)
if len(matchingLogs) < 1 {
err = transformer.Repository.MarkHeaderChecked(header.Id)
if err != nil {
return err
}
}
models, err := transformer.Converter.ToModels(matchingLogs)
if err != nil {
return err
}
err = transformer.Repository.Create(header.Id, models)
if err != nil {
return err
}
}
return nil
}

View File

@ -97,6 +97,42 @@ var _ = Describe("Drip drip transformer", func() {
Expect(err).To(MatchError(fakes.FakeError))
})
It("marks header checked if no logs returned", func() {
mockConverter := &drip_drip_mocks.MockDripDripConverter{}
mockRepository := &drip_drip_mocks.MockDripDripRepository{}
headerID := int64(123)
mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}})
mockFetcher := &mocks.MockLogFetcher{}
transformer := drip_drip.DripDripTransformer{
Converter: mockConverter,
Fetcher: mockFetcher,
Repository: mockRepository,
}
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
mockRepository.AssertMarkHeaderCheckedCalledWith(headerID)
})
It("returns error if marking header checked returns err", func() {
mockConverter := &drip_drip_mocks.MockDripDripConverter{}
mockRepository := &drip_drip_mocks.MockDripDripRepository{}
mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}})
mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError)
mockFetcher := &mocks.MockLogFetcher{}
transformer := drip_drip.DripDripTransformer{
Converter: mockConverter,
Fetcher: mockFetcher,
Repository: mockRepository,
}
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("converts matching logs", func() {
converter := &drip_drip_mocks.MockDripDripConverter{}
fetcher := &mocks.MockLogFetcher{}
@ -112,7 +148,7 @@ var _ = Describe("Drip drip transformer", func() {
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(converter.PassedLog).To(Equal(test_data.EthDripDripLog))
Expect(converter.PassedLogs).To(Equal([]types.Log{test_data.EthDripDripLog}))
})
It("returns error if converter returns error", func() {
@ -151,7 +187,7 @@ var _ = Describe("Drip drip transformer", func() {
Expect(err).NotTo(HaveOccurred())
Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id))
Expect(repository.PassedModel).To(Equal(test_data.DripDripModel))
Expect(repository.PassedModels).To(Equal([]drip_drip.DripDripModel{test_data.DripDripModel}))
})
It("returns error if repository returns error for create", func() {

View File

@ -25,28 +25,36 @@ import (
)
type Converter interface {
ToModel(ethLog types.Log) (DripFileIlkModel, error)
ToModels(ethLogs []types.Log) ([]DripFileIlkModel, error)
}
type DripFileIlkConverter struct{}
func (DripFileIlkConverter) ToModel(ethLog types.Log) (DripFileIlkModel, error) {
err := verifyLog(ethLog)
if err != nil {
return DripFileIlkModel{}, err
func (DripFileIlkConverter) ToModels(ethLogs []types.Log) ([]DripFileIlkModel, error) {
var models []DripFileIlkModel
for _, ethLog := range ethLogs {
err := verifyLog(ethLog)
if err != nil {
return nil, err
}
ilk := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
vow := string(bytes.Trim(ethLog.Topics[3].Bytes(), "\x00"))
taxBytes := ethLog.Data[len(ethLog.Data)-shared.DataItemLength:]
tax := big.NewInt(0).SetBytes(taxBytes).String()
raw, err := json.Marshal(ethLog)
if err != nil {
return nil, err
}
model := DripFileIlkModel{
Ilk: ilk,
Vow: vow,
Tax: tax,
TransactionIndex: ethLog.TxIndex,
Raw: raw,
}
models = append(models, model)
}
ilk := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
vow := string(bytes.Trim(ethLog.Topics[3].Bytes(), "\x00"))
taxBytes := ethLog.Data[len(ethLog.Data)-shared.DataItemLength:]
tax := big.NewInt(0).SetBytes(taxBytes).String()
raw, err := json.Marshal(ethLog)
return DripFileIlkModel{
Ilk: ilk,
Vow: vow,
Tax: tax,
TransactionIndex: ethLog.TxIndex,
Raw: raw,
}, err
return models, nil
}
func verifyLog(log types.Log) error {

View File

@ -32,7 +32,7 @@ var _ = Describe("Drip file ilk converter", func() {
Data: []byte{1, 1, 1, 1, 1},
}
_, err := converter.ToModel(badLog)
_, err := converter.ToModels([]types.Log{badLog})
Expect(err).To(HaveOccurred())
})
@ -43,7 +43,7 @@ var _ = Describe("Drip file ilk converter", func() {
Topics: []common.Hash{{}, {}, {}, {}},
}
_, err := converter.ToModel(badLog)
_, err := converter.ToModels([]types.Log{badLog})
Expect(err).To(HaveOccurred())
@ -52,9 +52,10 @@ var _ = Describe("Drip file ilk converter", func() {
It("converts a log to a model", func() {
converter := ilk.DripFileIlkConverter{}
model, err := converter.ToModel(test_data.EthDripFileIlkLog)
models, err := converter.ToModels([]types.Log{test_data.EthDripFileIlkLog})
Expect(err).NotTo(HaveOccurred())
Expect(model).To(Equal(test_data.DripFileIlkModel))
Expect(len(models)).To(Equal(1))
Expect(models[0]).To(Equal(test_data.DripFileIlkModel))
})
})

View File

@ -20,7 +20,8 @@ import (
)
type Repository interface {
Create(headerID int64, model DripFileIlkModel) error
Create(headerID int64, models []DripFileIlkModel) error
MarkHeaderChecked(headerID int64) error
MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
}
@ -32,12 +33,38 @@ func NewDripFileIlkRepository(db *postgres.DB) DripFileIlkRepository {
return DripFileIlkRepository{db: db}
}
func (repository DripFileIlkRepository) Create(headerID int64, model DripFileIlkModel) error {
_, err := repository.db.Exec(
`INSERT into maker.drip_file_ilk (header_id, ilk, vow, tax, tx_idx, raw_log)
func (repository DripFileIlkRepository) Create(headerID int64, models []DripFileIlkModel) error {
tx, err := repository.db.Begin()
if err != nil {
return err
}
for _, model := range models {
_, err = tx.Exec(
`INSERT into maker.drip_file_ilk (header_id, ilk, vow, tax, tx_idx, raw_log)
VALUES($1, $2, $3, $4::NUMERIC, $5, $6)`,
headerID, model.Ilk, model.Vow, model.Tax, model.TransactionIndex, model.Raw,
)
headerID, model.Ilk, model.Vow, model.Tax, model.TransactionIndex, model.Raw,
)
if err != nil {
tx.Rollback()
return err
}
}
_, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, drip_file_ilk_checked)
VALUES ($1, $2)
ON CONFLICT (header_id) DO
UPDATE SET drip_file_ilk_checked = $2`, headerID, true)
if err != nil {
tx.Rollback()
return err
}
return tx.Commit()
}
func (repository DripFileIlkRepository) MarkHeaderChecked(headerID int64) error {
_, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, drip_file_ilk_checked)
VALUES ($1, $2)
ON CONFLICT (header_id) DO
UPDATE SET drip_file_ilk_checked = $2`, headerID, true)
return err
}
@ -46,8 +73,8 @@ func (repository DripFileIlkRepository) MissingHeaders(startingBlockNumber, endi
err := repository.db.Select(
&result,
`SELECT headers.id, headers.block_number FROM headers
LEFT JOIN maker.drip_file_ilk on headers.id = header_id
WHERE header_id ISNULL
LEFT JOIN checked_headers on headers.id = header_id
WHERE (header_id ISNULL OR drip_file_ilk_checked IS FALSE)
AND headers.block_number >= $1
AND headers.block_number <= $2
AND headers.eth_node_fingerprint = $3`,

View File

@ -21,6 +21,7 @@ import (
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/ilk"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
@ -29,15 +30,24 @@ import (
var _ = Describe("Drip file ilk repository", func() {
Describe("Create", func() {
It("adds a drip file ilk event", func() {
db := test_config.NewTestDB(core.Node{})
var (
db *postgres.DB
dripFileIlkRepository ilk.Repository
err error
headerID int64
)
BeforeEach(func() {
db = test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
dripFileIlkRepository := ilk.NewDripFileIlkRepository(db)
dripFileIlkRepository = ilk.NewDripFileIlkRepository(db)
})
err = dripFileIlkRepository.Create(headerID, test_data.DripFileIlkModel)
It("adds a drip file ilk event", func() {
err = dripFileIlkRepository.Create(headerID, []ilk.DripFileIlkModel{test_data.DripFileIlkModel})
Expect(err).NotTo(HaveOccurred())
var dbDripFileIlk ilk.DripFileIlkModel
@ -50,30 +60,28 @@ var _ = Describe("Drip file ilk repository", func() {
Expect(dbDripFileIlk.Raw).To(MatchJSON(test_data.DripFileIlkModel.Raw))
})
It("does not duplicate drip file events", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
It("marks header as checked for logs", func() {
err = dripFileIlkRepository.Create(headerID, []ilk.DripFileIlkModel{test_data.DripFileIlkModel})
Expect(err).NotTo(HaveOccurred())
dripFileIlkRepository := ilk.NewDripFileIlkRepository(db)
err = dripFileIlkRepository.Create(headerID, test_data.DripFileIlkModel)
var headerChecked bool
err = db.Get(&headerChecked, `SELECT drip_file_ilk_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue())
})
It("does not duplicate drip file events", func() {
err = dripFileIlkRepository.Create(headerID, []ilk.DripFileIlkModel{test_data.DripFileIlkModel})
Expect(err).NotTo(HaveOccurred())
err = dripFileIlkRepository.Create(headerID, test_data.DripFileIlkModel)
err = dripFileIlkRepository.Create(headerID, []ilk.DripFileIlkModel{test_data.DripFileIlkModel})
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
})
It("removes drip file if corresponding header is deleted", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
dripFileIlkRepository := ilk.NewDripFileIlkRepository(db)
err = dripFileIlkRepository.Create(headerID, test_data.DripFileIlkModel)
err = dripFileIlkRepository.Create(headerID, []ilk.DripFileIlkModel{test_data.DripFileIlkModel})
Expect(err).NotTo(HaveOccurred())
_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
@ -86,6 +94,46 @@ var _ = Describe("Drip file ilk repository", func() {
})
})
Describe("MarkHeaderChecked", func() {
var (
db *postgres.DB
dripFileIlkRepository ilk.Repository
err error
headerID int64
)
BeforeEach(func() {
db = test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
dripFileIlkRepository = ilk.NewDripFileIlkRepository(db)
})
It("creates a row for a new headerID", func() {
err = dripFileIlkRepository.MarkHeaderChecked(headerID)
Expect(err).NotTo(HaveOccurred())
var headerChecked bool
err = db.Get(&headerChecked, `SELECT drip_file_ilk_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue())
})
It("updates row when headerID already exists", func() {
_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID)
err = dripFileIlkRepository.MarkHeaderChecked(headerID)
Expect(err).NotTo(HaveOccurred())
var headerChecked bool
err = db.Get(&headerChecked, `SELECT drip_file_ilk_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue())
})
})
Describe("MissingHeaders", func() {
It("returns headers with no associated drip file event", func() {
db := test_config.NewTestDB(core.Node{})
@ -102,7 +150,7 @@ var _ = Describe("Drip file ilk repository", func() {
Expect(err).NotTo(HaveOccurred())
}
dripFileIlkRepository := ilk.NewDripFileIlkRepository(db)
err := dripFileIlkRepository.Create(headerIDs[1], test_data.DripFileIlkModel)
err := dripFileIlkRepository.MarkHeaderChecked(headerIDs[1])
Expect(err).NotTo(HaveOccurred())
headers, err := dripFileIlkRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
@ -113,6 +161,33 @@ var _ = Describe("Drip file ilk repository", func() {
Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
})
It("only treats headers as checked if drip file ilk logs have been checked", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
startingBlockNumber := int64(1)
dripFileIlkdBlockNumber := int64(2)
endingBlockNumber := int64(3)
blockNumbers := []int64{startingBlockNumber, dripFileIlkdBlockNumber, endingBlockNumber, endingBlockNumber + 1}
var headerIDs []int64
for _, n := range blockNumbers {
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
headerIDs = append(headerIDs, headerID)
Expect(err).NotTo(HaveOccurred())
}
dripFileIlkRepository := ilk.NewDripFileIlkRepository(db)
_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
Expect(err).NotTo(HaveOccurred())
headers, err := dripFileIlkRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
Expect(err).NotTo(HaveOccurred())
Expect(len(headers)).To(Equal(3))
Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripFileIlkdBlockNumber)))
Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripFileIlkdBlockNumber)))
Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripFileIlkdBlockNumber)))
})
It("only returns headers associated with the current node", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
@ -130,7 +205,7 @@ var _ = Describe("Drip file ilk repository", func() {
}
dripFileIlkRepository := ilk.NewDripFileIlkRepository(db)
dripFileIlkRepositoryTwo := ilk.NewDripFileIlkRepository(dbTwo)
err := dripFileIlkRepository.Create(headerIDs[0], test_data.DripFileIlkModel)
err := dripFileIlkRepository.MarkHeaderChecked(headerIDs[0])
Expect(err).NotTo(HaveOccurred())
nodeOneMissingHeaders, err := dripFileIlkRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])

View File

@ -60,16 +60,20 @@ func (transformer DripFileIlkTransformer) Execute() error {
if err != nil {
return err
}
for _, log := range matchingLogs {
model, err := transformer.Converter.ToModel(log)
if err != nil {
return err
}
err = transformer.Repository.Create(header.Id, model)
if len(matchingLogs) < 1 {
err = transformer.Repository.MarkHeaderChecked(header.Id)
if err != nil {
return err
}
}
model, err := transformer.Converter.ToModels(matchingLogs)
if err != nil {
return err
}
err = transformer.Repository.Create(header.Id, model)
if err != nil {
return err
}
}
return nil
}

View File

@ -19,6 +19,7 @@ import (
"github.com/ethereum/go-ethereum/core/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file"
@ -96,6 +97,42 @@ var _ = Describe("Drip file ilk transformer", func() {
Expect(err).To(MatchError(fakes.FakeError))
})
It("marks header checked if no logs returned", func() {
mockConverter := &ilk_mocks.MockDripFileIlkConverter{}
mockRepository := &ilk_mocks.MockDripFileIlkRepository{}
headerID := int64(123)
mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}})
mockFetcher := &mocks.MockLogFetcher{}
transformer := ilk.DripFileIlkTransformer{
Converter: mockConverter,
Fetcher: mockFetcher,
Repository: mockRepository,
}
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
mockRepository.AssertMarkHeaderCheckedCalledWith(headerID)
})
It("returns error if marking header checked returns err", func() {
mockConverter := &ilk_mocks.MockDripFileIlkConverter{}
mockRepository := &ilk_mocks.MockDripFileIlkRepository{}
mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}})
mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError)
mockFetcher := &mocks.MockLogFetcher{}
transformer := ilk.DripFileIlkTransformer{
Converter: mockConverter,
Fetcher: mockFetcher,
Repository: mockRepository,
}
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("converts matching logs", func() {
converter := &ilk_mocks.MockDripFileIlkConverter{}
fetcher := &mocks.MockLogFetcher{}
@ -111,7 +148,7 @@ var _ = Describe("Drip file ilk transformer", func() {
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(converter.PassedLog).To(Equal(test_data.EthDripFileIlkLog))
Expect(converter.PassedLogs).To(Equal([]types.Log{test_data.EthDripFileIlkLog}))
})
It("returns error if converter returns error", func() {
@ -150,7 +187,7 @@ var _ = Describe("Drip file ilk transformer", func() {
Expect(err).NotTo(HaveOccurred())
Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id))
Expect(repository.PassedModel).To(Equal(test_data.DripFileIlkModel))
Expect(repository.PassedModels).To(Equal([]ilk.DripFileIlkModel{test_data.DripFileIlkModel}))
})
It("returns error if repository returns error for create", func() {

View File

@ -23,25 +23,33 @@ import (
)
type Converter interface {
ToModel(ethLog types.Log) (DripFileRepoModel, error)
ToModels(ethLogs []types.Log) ([]DripFileRepoModel, error)
}
type DripFileRepoConverter struct{}
func (DripFileRepoConverter) ToModel(ethLog types.Log) (DripFileRepoModel, error) {
err := verifyLog(ethLog)
if err != nil {
return DripFileRepoModel{}, err
func (DripFileRepoConverter) ToModels(ethLogs []types.Log) ([]DripFileRepoModel, error) {
var models []DripFileRepoModel
for _, ethLog := range ethLogs {
err := verifyLog(ethLog)
if err != nil {
return nil, err
}
what := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
data := big.NewInt(0).SetBytes(ethLog.Topics[3].Bytes()).String()
raw, err := json.Marshal(ethLog)
if err != nil {
return nil, err
}
model := DripFileRepoModel{
What: what,
Data: data,
TransactionIndex: ethLog.TxIndex,
Raw: raw,
}
models = append(models, model)
}
what := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
data := big.NewInt(0).SetBytes(ethLog.Topics[3].Bytes()).String()
raw, err := json.Marshal(ethLog)
return DripFileRepoModel{
What: what,
Data: data,
TransactionIndex: ethLog.TxIndex,
Raw: raw,
}, err
return models, nil
}
func verifyLog(log types.Log) error {

View File

@ -32,7 +32,7 @@ var _ = Describe("Drip file repo converter", func() {
Data: []byte{1, 1, 1, 1, 1},
}
_, err := converter.ToModel(badLog)
_, err := converter.ToModels([]types.Log{badLog})
Expect(err).To(HaveOccurred())
})
@ -40,9 +40,10 @@ var _ = Describe("Drip file repo converter", func() {
It("converts a log to a model", func() {
converter := repo.DripFileRepoConverter{}
model, err := converter.ToModel(test_data.EthDripFileRepoLog)
models, err := converter.ToModels([]types.Log{test_data.EthDripFileRepoLog})
Expect(err).NotTo(HaveOccurred())
Expect(model).To(Equal(test_data.DripFileRepoModel))
Expect(len(models)).To(Equal(1))
Expect(models[0]).To(Equal(test_data.DripFileRepoModel))
})
})

View File

@ -20,7 +20,8 @@ import (
)
type Repository interface {
Create(headerID int64, model DripFileRepoModel) error
Create(headerID int64, models []DripFileRepoModel) error
MarkHeaderChecked(headerID int64) error
MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
}
@ -32,12 +33,38 @@ func NewDripFileRepoRepository(db *postgres.DB) DripFileRepoRepository {
return DripFileRepoRepository{db: db}
}
func (repository DripFileRepoRepository) Create(headerID int64, model DripFileRepoModel) error {
_, err := repository.db.Exec(
`INSERT into maker.drip_file_repo (header_id, what, data, tx_idx, raw_log)
func (repository DripFileRepoRepository) Create(headerID int64, models []DripFileRepoModel) error {
tx, err := repository.db.Begin()
if err != nil {
return err
}
for _, model := range models {
_, err = tx.Exec(
`INSERT into maker.drip_file_repo (header_id, what, data, tx_idx, raw_log)
VALUES($1, $2, $3::NUMERIC, $4, $5)`,
headerID, model.What, model.Data, model.TransactionIndex, model.Raw,
)
headerID, model.What, model.Data, model.TransactionIndex, model.Raw,
)
if err != nil {
tx.Rollback()
return err
}
}
_, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, drip_file_repo_checked)
VALUES ($1, $2)
ON CONFLICT (header_id) DO
UPDATE SET drip_file_repo_checked = $2`, headerID, true)
if err != nil {
tx.Rollback()
return err
}
return tx.Commit()
}
func (repository DripFileRepoRepository) MarkHeaderChecked(headerID int64) error {
_, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, drip_file_repo_checked)
VALUES ($1, $2)
ON CONFLICT (header_id) DO
UPDATE SET drip_file_repo_checked = $2`, headerID, true)
return err
}
@ -46,8 +73,8 @@ func (repository DripFileRepoRepository) MissingHeaders(startingBlockNumber, end
err := repository.db.Select(
&result,
`SELECT headers.id, headers.block_number FROM headers
LEFT JOIN maker.drip_file_repo on headers.id = header_id
WHERE header_id ISNULL
LEFT JOIN checked_headers on headers.id = header_id
WHERE (header_id ISNULL OR drip_file_repo_checked IS FALSE)
AND headers.block_number >= $1
AND headers.block_number <= $2
AND headers.eth_node_fingerprint = $3`,

View File

@ -19,23 +19,33 @@ import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/repo"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
"github.com/vulcanize/vulcanizedb/test_config"
)
var _ = Describe("", func() {
var _ = Describe("Drip file repo repository", func() {
Describe("Create", func() {
It("adds a drip file repo event", func() {
db := test_config.NewTestDB(core.Node{})
var (
db *postgres.DB
dripFileRepoRepository repo.Repository
err error
headerID int64
)
BeforeEach(func() {
db = test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
dripFileRepoRepository := repo.NewDripFileRepoRepository(db)
dripFileRepoRepository = repo.NewDripFileRepoRepository(db)
})
err = dripFileRepoRepository.Create(headerID, test_data.DripFileRepoModel)
It("adds a drip file repo event", func() {
err = dripFileRepoRepository.Create(headerID, []repo.DripFileRepoModel{test_data.DripFileRepoModel})
Expect(err).NotTo(HaveOccurred())
var dbDripFileRepo repo.DripFileRepoModel
@ -47,30 +57,28 @@ var _ = Describe("", func() {
Expect(dbDripFileRepo.Raw).To(MatchJSON(test_data.DripFileRepoModel.Raw))
})
It("does not duplicate drip file events", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
It("marks header as checked for logs", func() {
err = dripFileRepoRepository.Create(headerID, []repo.DripFileRepoModel{test_data.DripFileRepoModel})
Expect(err).NotTo(HaveOccurred())
dripFileRepoRepository := repo.NewDripFileRepoRepository(db)
err = dripFileRepoRepository.Create(headerID, test_data.DripFileRepoModel)
var headerChecked bool
err = db.Get(&headerChecked, `SELECT drip_file_repo_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue())
})
It("does not duplicate drip file events", func() {
err = dripFileRepoRepository.Create(headerID, []repo.DripFileRepoModel{test_data.DripFileRepoModel})
Expect(err).NotTo(HaveOccurred())
err = dripFileRepoRepository.Create(headerID, test_data.DripFileRepoModel)
err = dripFileRepoRepository.Create(headerID, []repo.DripFileRepoModel{test_data.DripFileRepoModel})
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
})
It("removes drip file if corresponding header is deleted", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
dripFileRepoRepository := repo.NewDripFileRepoRepository(db)
err = dripFileRepoRepository.Create(headerID, test_data.DripFileRepoModel)
err = dripFileRepoRepository.Create(headerID, []repo.DripFileRepoModel{test_data.DripFileRepoModel})
Expect(err).NotTo(HaveOccurred())
_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
@ -83,6 +91,46 @@ var _ = Describe("", func() {
})
})
Describe("MarkHeaderChecked", func() {
var (
db *postgres.DB
dripFileRepoRepository repo.Repository
err error
headerID int64
)
BeforeEach(func() {
db = test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
dripFileRepoRepository = repo.NewDripFileRepoRepository(db)
})
It("creates a row for a new headerID", func() {
err = dripFileRepoRepository.MarkHeaderChecked(headerID)
Expect(err).NotTo(HaveOccurred())
var headerChecked bool
err = db.Get(&headerChecked, `SELECT drip_file_repo_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue())
})
It("updates row when headerID already exists", func() {
_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID)
err = dripFileRepoRepository.MarkHeaderChecked(headerID)
Expect(err).NotTo(HaveOccurred())
var headerChecked bool
err = db.Get(&headerChecked, `SELECT drip_file_repo_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue())
})
})
Describe("MissingHeaders", func() {
It("returns headers with no associated drip file event", func() {
db := test_config.NewTestDB(core.Node{})
@ -99,7 +147,7 @@ var _ = Describe("", func() {
Expect(err).NotTo(HaveOccurred())
}
dripFileRepoRepository := repo.NewDripFileRepoRepository(db)
err := dripFileRepoRepository.Create(headerIDs[1], test_data.DripFileRepoModel)
err := dripFileRepoRepository.MarkHeaderChecked(headerIDs[1])
Expect(err).NotTo(HaveOccurred())
headers, err := dripFileRepoRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
@ -110,6 +158,33 @@ var _ = Describe("", func() {
Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
})
It("only treats headers as checked if drip file repo logs have been checked", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
startingBlockNumber := int64(1)
dripFileRepodBlockNumber := int64(2)
endingBlockNumber := int64(3)
blockNumbers := []int64{startingBlockNumber, dripFileRepodBlockNumber, endingBlockNumber, endingBlockNumber + 1}
var headerIDs []int64
for _, n := range blockNumbers {
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
headerIDs = append(headerIDs, headerID)
Expect(err).NotTo(HaveOccurred())
}
dripFileRepoRepository := repo.NewDripFileRepoRepository(db)
_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
Expect(err).NotTo(HaveOccurred())
headers, err := dripFileRepoRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
Expect(err).NotTo(HaveOccurred())
Expect(len(headers)).To(Equal(3))
Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripFileRepodBlockNumber)))
Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripFileRepodBlockNumber)))
Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripFileRepodBlockNumber)))
})
It("only returns headers associated with the current node", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
@ -127,7 +202,7 @@ var _ = Describe("", func() {
}
dripFileRepoRepository := repo.NewDripFileRepoRepository(db)
dripFileRepoRepositoryTwo := repo.NewDripFileRepoRepository(dbTwo)
err := dripFileRepoRepository.Create(headerIDs[0], test_data.DripFileRepoModel)
err := dripFileRepoRepository.MarkHeaderChecked(headerIDs[0])
Expect(err).NotTo(HaveOccurred())
nodeOneMissingHeaders, err := dripFileRepoRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])

View File

@ -60,16 +60,20 @@ func (transformer DripFileRepoTransformer) Execute() error {
if err != nil {
return err
}
for _, log := range matchingLogs {
model, err := transformer.Converter.ToModel(log)
if err != nil {
return err
}
err = transformer.Repository.Create(header.Id, model)
if len(matchingLogs) < 1 {
err = transformer.Repository.MarkHeaderChecked(header.Id)
if err != nil {
return err
}
}
model, err := transformer.Converter.ToModels(matchingLogs)
if err != nil {
return err
}
err = transformer.Repository.Create(header.Id, model)
if err != nil {
return err
}
}
return nil
}

View File

@ -97,6 +97,42 @@ var _ = Describe("Drip file repo transformer", func() {
Expect(err).To(MatchError(fakes.FakeError))
})
It("marks header checked if no logs returned", func() {
mockConverter := &repo_mocks.MockDripFileRepoConverter{}
mockRepository := &repo_mocks.MockDripFileRepoRepository{}
headerID := int64(123)
mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}})
mockFetcher := &mocks.MockLogFetcher{}
transformer := repo.DripFileRepoTransformer{
Converter: mockConverter,
Fetcher: mockFetcher,
Repository: mockRepository,
}
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
mockRepository.AssertMarkHeaderCheckedCalledWith(headerID)
})
It("returns error if marking header checked returns err", func() {
mockConverter := &repo_mocks.MockDripFileRepoConverter{}
mockRepository := &repo_mocks.MockDripFileRepoRepository{}
mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}})
mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError)
mockFetcher := &mocks.MockLogFetcher{}
transformer := repo.DripFileRepoTransformer{
Converter: mockConverter,
Fetcher: mockFetcher,
Repository: mockRepository,
}
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("converts matching logs", func() {
converter := &repo_mocks.MockDripFileRepoConverter{}
fetcher := &mocks.MockLogFetcher{}
@ -112,7 +148,7 @@ var _ = Describe("Drip file repo transformer", func() {
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(converter.PassedLog).To(Equal(test_data.EthDripFileRepoLog))
Expect(converter.PassedLogs).To(Equal([]types.Log{test_data.EthDripFileRepoLog}))
})
It("returns error if converter returns error", func() {
@ -151,7 +187,7 @@ var _ = Describe("Drip file repo transformer", func() {
Expect(err).NotTo(HaveOccurred())
Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id))
Expect(repository.PassedModel).To(Equal(test_data.DripFileRepoModel))
Expect(repository.PassedModels).To(Equal([]repo.DripFileRepoModel{test_data.DripFileRepoModel}))
})
It("returns error if repository returns error for create", func() {

View File

@ -22,25 +22,33 @@ import (
)
type Converter interface {
ToModel(ethLog types.Log) (DripFileVowModel, error)
ToModels(ethLogs []types.Log) ([]DripFileVowModel, error)
}
type DripFileVowConverter struct{}
func (DripFileVowConverter) ToModel(ethLog types.Log) (DripFileVowModel, error) {
err := verifyLog(ethLog)
if err != nil {
return DripFileVowModel{}, err
func (DripFileVowConverter) ToModels(ethLogs []types.Log) ([]DripFileVowModel, error) {
var models []DripFileVowModel
for _, ethLog := range ethLogs {
err := verifyLog(ethLog)
if err != nil {
return nil, err
}
what := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
data := string(bytes.Trim(ethLog.Topics[3].Bytes(), "\x00"))
raw, err := json.Marshal(ethLog)
if err != nil {
return nil, err
}
model := DripFileVowModel{
What: what,
Data: data,
TransactionIndex: ethLog.TxIndex,
Raw: raw,
}
models = append(models, model)
}
what := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
data := string(bytes.Trim(ethLog.Topics[3].Bytes(), "\x00"))
raw, err := json.Marshal(ethLog)
return DripFileVowModel{
What: what,
Data: data,
TransactionIndex: ethLog.TxIndex,
Raw: raw,
}, err
return models, nil
}
func verifyLog(log types.Log) error {

View File

@ -32,7 +32,7 @@ var _ = Describe("Drip file repo converter", func() {
Data: []byte{1, 1, 1, 1, 1},
}
_, err := converter.ToModel(badLog)
_, err := converter.ToModels([]types.Log{badLog})
Expect(err).To(HaveOccurred())
})
@ -40,9 +40,10 @@ var _ = Describe("Drip file repo converter", func() {
It("converts a log to a model", func() {
converter := vow.DripFileVowConverter{}
model, err := converter.ToModel(test_data.EthDripFileVowLog)
models, err := converter.ToModels([]types.Log{test_data.EthDripFileVowLog})
Expect(err).NotTo(HaveOccurred())
Expect(model).To(Equal(test_data.DripFileVowModel))
Expect(len(models)).To(Equal(1))
Expect(models[0]).To(Equal(test_data.DripFileVowModel))
})
})

View File

@ -20,7 +20,8 @@ import (
)
type Repository interface {
Create(headerID int64, model DripFileVowModel) error
Create(headerID int64, models []DripFileVowModel) error
MarkHeaderChecked(headerID int64) error
MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
}
@ -32,12 +33,38 @@ func NewDripFileVowRepository(db *postgres.DB) DripFileVowRepository {
return DripFileVowRepository{db: db}
}
func (repository DripFileVowRepository) Create(headerID int64, model DripFileVowModel) error {
_, err := repository.db.Exec(
`INSERT into maker.drip_file_vow (header_id, what, data, tx_idx, raw_log)
func (repository DripFileVowRepository) Create(headerID int64, models []DripFileVowModel) error {
tx, err := repository.db.Begin()
if err != nil {
return err
}
for _, model := range models {
_, err = tx.Exec(
`INSERT into maker.drip_file_vow (header_id, what, data, tx_idx, raw_log)
VALUES($1, $2, $3, $4, $5)`,
headerID, model.What, model.Data, model.TransactionIndex, model.Raw,
)
headerID, model.What, model.Data, model.TransactionIndex, model.Raw,
)
if err != nil {
tx.Rollback()
return err
}
}
_, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, drip_file_vow_checked)
VALUES ($1, $2)
ON CONFLICT (header_id) DO
UPDATE SET drip_file_vow_checked = $2`, headerID, true)
if err != nil {
tx.Rollback()
return err
}
return tx.Commit()
}
func (repository DripFileVowRepository) MarkHeaderChecked(headerID int64) error {
_, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, drip_file_vow_checked)
VALUES ($1, $2)
ON CONFLICT (header_id) DO
UPDATE SET drip_file_vow_checked = $2`, headerID, true)
return err
}
@ -46,8 +73,8 @@ func (repository DripFileVowRepository) MissingHeaders(startingBlockNumber, endi
err := repository.db.Select(
&result,
`SELECT headers.id, headers.block_number FROM headers
LEFT JOIN maker.drip_file_vow on headers.id = header_id
WHERE header_id ISNULL
LEFT JOIN checked_headers on headers.id = header_id
WHERE (header_id ISNULL OR drip_file_vow_checked IS FALSE)
AND headers.block_number >= $1
AND headers.block_number <= $2
AND headers.eth_node_fingerprint = $3`,

View File

@ -16,26 +16,38 @@ package vow_test
import (
"database/sql"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/vow"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
"github.com/vulcanize/vulcanizedb/test_config"
)
var _ = Describe("", func() {
var _ = Describe("Drip file vow repository", func() {
Describe("Create", func() {
It("adds a drip file vow event", func() {
db := test_config.NewTestDB(core.Node{})
var (
db *postgres.DB
dripFileVowRepository vow.Repository
err error
headerID int64
)
BeforeEach(func() {
db = test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
dripFileVowRepository := vow.NewDripFileVowRepository(db)
dripFileVowRepository = vow.NewDripFileVowRepository(db)
})
err = dripFileVowRepository.Create(headerID, test_data.DripFileVowModel)
It("adds a drip file vow event", func() {
err = dripFileVowRepository.Create(headerID, []vow.DripFileVowModel{test_data.DripFileVowModel})
Expect(err).NotTo(HaveOccurred())
var dbDripFileVow vow.DripFileVowModel
@ -47,30 +59,28 @@ var _ = Describe("", func() {
Expect(dbDripFileVow.Raw).To(MatchJSON(test_data.DripFileVowModel.Raw))
})
It("does not duplicate drip file events", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
It("marks header as checked for logs", func() {
err = dripFileVowRepository.Create(headerID, []vow.DripFileVowModel{test_data.DripFileVowModel})
Expect(err).NotTo(HaveOccurred())
dripFileVowRepository := vow.NewDripFileVowRepository(db)
err = dripFileVowRepository.Create(headerID, test_data.DripFileVowModel)
var headerChecked bool
err = db.Get(&headerChecked, `SELECT drip_file_vow_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue())
})
It("does not duplicate drip file events", func() {
err = dripFileVowRepository.Create(headerID, []vow.DripFileVowModel{test_data.DripFileVowModel})
Expect(err).NotTo(HaveOccurred())
err = dripFileVowRepository.Create(headerID, test_data.DripFileVowModel)
err = dripFileVowRepository.Create(headerID, []vow.DripFileVowModel{test_data.DripFileVowModel})
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
})
It("removes drip file if corresponding header is deleted", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
dripFileVowRepository := vow.NewDripFileVowRepository(db)
err = dripFileVowRepository.Create(headerID, test_data.DripFileVowModel)
err = dripFileVowRepository.Create(headerID, []vow.DripFileVowModel{test_data.DripFileVowModel})
Expect(err).NotTo(HaveOccurred())
_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
@ -83,8 +93,48 @@ var _ = Describe("", func() {
})
})
Describe("MarkHeaderChecked", func() {
var (
db *postgres.DB
dripFileVowRepository vow.Repository
err error
headerID int64
)
BeforeEach(func() {
db = test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
dripFileVowRepository = vow.NewDripFileVowRepository(db)
})
It("creates a row for a new headerID", func() {
err = dripFileVowRepository.MarkHeaderChecked(headerID)
Expect(err).NotTo(HaveOccurred())
var headerChecked bool
err = db.Get(&headerChecked, `SELECT drip_file_vow_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue())
})
It("updates row when headerID already exists", func() {
_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID)
err = dripFileVowRepository.MarkHeaderChecked(headerID)
Expect(err).NotTo(HaveOccurred())
var headerChecked bool
err = db.Get(&headerChecked, `SELECT drip_file_vow_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue())
})
})
Describe("MissingHeaders", func() {
It("returns headers with no associated drip file event", func() {
It("returns headers that haven't been checked", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
@ -99,7 +149,7 @@ var _ = Describe("", func() {
Expect(err).NotTo(HaveOccurred())
}
dripFileVowRepository := vow.NewDripFileVowRepository(db)
err := dripFileVowRepository.Create(headerIDs[1], test_data.DripFileVowModel)
err := dripFileVowRepository.MarkHeaderChecked(headerIDs[1])
Expect(err).NotTo(HaveOccurred())
headers, err := dripFileVowRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
@ -110,6 +160,33 @@ var _ = Describe("", func() {
Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
})
It("only treats headers as checked if drip file vow logs have been checked", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
startingBlockNumber := int64(1)
dripFileVowdBlockNumber := int64(2)
endingBlockNumber := int64(3)
blockNumbers := []int64{startingBlockNumber, dripFileVowdBlockNumber, endingBlockNumber, endingBlockNumber + 1}
var headerIDs []int64
for _, n := range blockNumbers {
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
headerIDs = append(headerIDs, headerID)
Expect(err).NotTo(HaveOccurred())
}
dripFileVowRepository := vow.NewDripFileVowRepository(db)
_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
Expect(err).NotTo(HaveOccurred())
headers, err := dripFileVowRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
Expect(err).NotTo(HaveOccurred())
Expect(len(headers)).To(Equal(3))
Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripFileVowdBlockNumber)))
Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripFileVowdBlockNumber)))
Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripFileVowdBlockNumber)))
})
It("only returns headers associated with the current node", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
@ -127,7 +204,7 @@ var _ = Describe("", func() {
}
dripFileVowRepository := vow.NewDripFileVowRepository(db)
dripFileVowRepositoryTwo := vow.NewDripFileVowRepository(dbTwo)
err := dripFileVowRepository.Create(headerIDs[0], test_data.DripFileVowModel)
err := dripFileVowRepository.MarkHeaderChecked(headerIDs[0])
Expect(err).NotTo(HaveOccurred())
nodeOneMissingHeaders, err := dripFileVowRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])

View File

@ -60,16 +60,20 @@ func (transformer DripFileVowTransformer) Execute() error {
if err != nil {
return err
}
for _, log := range matchingLogs {
model, err := transformer.Converter.ToModel(log)
if err != nil {
return err
}
err = transformer.Repository.Create(header.Id, model)
if len(matchingLogs) < 1 {
err = transformer.Repository.MarkHeaderChecked(header.Id)
if err != nil {
return err
}
}
model, err := transformer.Converter.ToModels(matchingLogs)
if err != nil {
return err
}
err = transformer.Repository.Create(header.Id, model)
if err != nil {
return err
}
}
return nil
}

View File

@ -97,6 +97,42 @@ var _ = Describe("Drip file vow transformer", func() {
Expect(err).To(MatchError(fakes.FakeError))
})
It("marks header checked if no logs returned", func() {
mockConverter := &vow_mocks.MockDripFileVowConverter{}
mockRepository := &vow_mocks.MockDripFileVowRepository{}
headerID := int64(123)
mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}})
mockFetcher := &mocks.MockLogFetcher{}
transformer := vow.DripFileVowTransformer{
Converter: mockConverter,
Fetcher: mockFetcher,
Repository: mockRepository,
}
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
mockRepository.AssertMarkHeaderCheckedCalledWith(headerID)
})
It("returns error if marking header checked returns err", func() {
mockConverter := &vow_mocks.MockDripFileVowConverter{}
mockRepository := &vow_mocks.MockDripFileVowRepository{}
mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}})
mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError)
mockFetcher := &mocks.MockLogFetcher{}
transformer := vow.DripFileVowTransformer{
Converter: mockConverter,
Fetcher: mockFetcher,
Repository: mockRepository,
}
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("converts matching logs", func() {
converter := &vow_mocks.MockDripFileVowConverter{}
fetcher := &mocks.MockLogFetcher{}
@ -112,7 +148,7 @@ var _ = Describe("Drip file vow transformer", func() {
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(converter.PassedLog).To(Equal(test_data.EthDripFileVowLog))
Expect(converter.PassedLogs).To(Equal([]types.Log{test_data.EthDripFileVowLog}))
})
It("returns error if converter returns error", func() {
@ -151,7 +187,7 @@ var _ = Describe("Drip file vow transformer", func() {
Expect(err).NotTo(HaveOccurred())
Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id))
Expect(repository.PassedModel).To(Equal(test_data.DripFileVowModel))
Expect(repository.PassedModels).To(Equal([]vow.DripFileVowModel{test_data.DripFileVowModel}))
})
It("returns error if repository returns error for create", func() {

View File

@ -16,7 +16,6 @@ package frob
import (
"github.com/ethereum/go-ethereum/accounts/abi/bind"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"encoding/json"
@ -24,40 +23,50 @@ import (
)
type Converter interface {
ToEntity(contractAddress string, contractAbi string, ethLog types.Log) (FrobEntity, error)
ToModel(flipKick FrobEntity) (FrobModel, error)
ToEntities(contractAbi string, ethLogs []types.Log) ([]FrobEntity, error)
ToModels(entities []FrobEntity) ([]FrobModel, error)
}
type FrobConverter struct{}
func (FrobConverter) ToEntity(contractAddress string, contractAbi string, ethLog types.Log) (FrobEntity, error) {
entity := FrobEntity{}
address := common.HexToAddress(contractAddress)
abi, err := geth.ParseAbi(contractAbi)
if err != nil {
return entity, err
func (FrobConverter) ToEntities(contractAbi string, ethLogs []types.Log) ([]FrobEntity, error) {
var entities []FrobEntity
for _, ethLog := range ethLogs {
entity := FrobEntity{}
address := ethLog.Address
abi, err := geth.ParseAbi(contractAbi)
if err != nil {
return nil, err
}
contract := bind.NewBoundContract(address, abi, nil, nil, nil)
err = contract.UnpackLog(&entity, "Frob", ethLog)
entity.TransactionIndex = ethLog.TxIndex
entity.Raw = ethLog
entities = append(entities, entity)
}
contract := bind.NewBoundContract(address, abi, nil, nil, nil)
err = contract.UnpackLog(&entity, "Frob", ethLog)
entity.TransactionIndex = ethLog.TxIndex
entity.Raw = ethLog
return entity, err
return entities, nil
}
func (FrobConverter) ToModel(frob FrobEntity) (FrobModel, error) {
rawLog, err := json.Marshal(frob.Raw)
if err != nil {
return FrobModel{}, err
func (FrobConverter) ToModels(entities []FrobEntity) ([]FrobModel, error) {
var models []FrobModel
for _, entity := range entities {
rawLog, err := json.Marshal(entity.Raw)
if err != nil {
return nil, err
}
model := FrobModel{
Ilk: entity.Ilk[:],
Urn: entity.Urn[:],
Ink: entity.Ink.String(),
Art: entity.Art.String(),
Dink: entity.Dink.String(),
Dart: entity.Dart.String(),
IArt: entity.IArt.String(),
TransactionIndex: entity.TransactionIndex,
Raw: rawLog,
}
models = append(models, model)
}
return FrobModel{
Ilk: frob.Ilk[:],
Urn: frob.Urn[:],
Ink: frob.Ink.String(),
Art: frob.Art.String(),
Dink: frob.Dink.String(),
Dart: frob.Dart.String(),
IArt: frob.IArt.String(),
TransactionIndex: frob.TransactionIndex,
Raw: rawLog,
}, nil
return models, nil
}

View File

@ -18,6 +18,7 @@ import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/transformers/frob"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
@ -27,18 +28,20 @@ var _ = Describe("Frob converter", func() {
It("converts a log to an entity", func() {
converter := frob.FrobConverter{}
entity, err := converter.ToEntity(shared.PitContractAddress, shared.PitABI, test_data.EthFrobLog)
entities, err := converter.ToEntities(shared.PitABI, []types.Log{test_data.EthFrobLog})
Expect(err).NotTo(HaveOccurred())
Expect(entity).To(Equal(test_data.FrobEntity))
Expect(len(entities)).To(Equal(1))
Expect(entities[0]).To(Equal(test_data.FrobEntity))
})
It("converts an entity to a model", func() {
converter := frob.FrobConverter{}
model, err := converter.ToModel(test_data.FrobEntity)
models, err := converter.ToModels([]frob.FrobEntity{test_data.FrobEntity})
Expect(err).NotTo(HaveOccurred())
Expect(model).To(Equal(test_data.FrobModel))
Expect(len(models)).To(Equal(1))
Expect(models[0]).To(Equal(test_data.FrobModel))
})
})

View File

@ -20,7 +20,8 @@ import (
)
type Repository interface {
Create(headerID int64, model FrobModel) error
Create(headerID int64, models []FrobModel) error
MarkHeaderChecked(headerID int64) error
MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
}
@ -32,10 +33,36 @@ func NewFrobRepository(db *postgres.DB) FrobRepository {
return FrobRepository{db: db}
}
func (repository FrobRepository) Create(headerID int64, model FrobModel) error {
_, err := repository.db.Exec(`INSERT INTO maker.frob (header_id, art, dart, dink, iart, ilk, ink, urn, raw_log, tx_idx)
func (repository FrobRepository) Create(headerID int64, models []FrobModel) error {
tx, err := repository.db.Begin()
if err != nil {
return err
}
for _, model := range models {
_, err = tx.Exec(`INSERT INTO maker.frob (header_id, art, dart, dink, iart, ilk, ink, urn, raw_log, tx_idx)
VALUES($1, $2::NUMERIC, $3::NUMERIC, $4::NUMERIC, $5::NUMERIC, $6, $7::NUMERIC, $8, $9, $10)`,
headerID, model.Art, model.Dart, model.Dink, model.IArt, model.Ilk, model.Ink, model.Urn, model.Raw, model.TransactionIndex)
headerID, model.Art, model.Dart, model.Dink, model.IArt, model.Ilk, model.Ink, model.Urn, model.Raw, model.TransactionIndex)
if err != nil {
tx.Rollback()
return err
}
}
_, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, frob_checked)
VALUES ($1, $2)
ON CONFLICT (header_id) DO
UPDATE SET frob_checked = $2`, headerID, true)
if err != nil {
tx.Rollback()
return err
}
return tx.Commit()
}
func (repository FrobRepository) MarkHeaderChecked(headerID int64) error {
_, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, frob_checked)
VALUES ($1, $2)
ON CONFLICT (header_id) DO
UPDATE SET frob_checked = $2`, headerID, true)
return err
}
@ -44,8 +71,8 @@ func (repository FrobRepository) MissingHeaders(startingBlockNumber, endingBlock
err := repository.db.Select(
&result,
`SELECT headers.id, headers.block_number FROM headers
LEFT JOIN maker.frob on headers.id = header_id
WHERE header_id ISNULL
LEFT JOIN checked_headers on headers.id = header_id
WHERE (header_id ISNULL OR frob_checked IS FALSE)
AND headers.block_number >= $1
AND headers.block_number <= $2
AND headers.eth_node_fingerprint = $3`,

View File

@ -21,6 +21,7 @@ import (
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/frob"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
@ -29,15 +30,24 @@ import (
var _ = Describe("Frob repository", func() {
Describe("Create", func() {
It("adds a frob", func() {
db := test_config.NewTestDB(core.Node{})
var (
db *postgres.DB
frobRepository frob.Repository
err error
headerID int64
)
BeforeEach(func() {
db = test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
frobRepository := frob.NewFrobRepository(db)
frobRepository = frob.NewFrobRepository(db)
})
err = frobRepository.Create(headerID, test_data.FrobModel)
It("adds a frob", func() {
err = frobRepository.Create(headerID, []frob.FrobModel{test_data.FrobModel})
Expect(err).NotTo(HaveOccurred())
var dbFrob frob.FrobModel
@ -54,30 +64,28 @@ var _ = Describe("Frob repository", func() {
Expect(dbFrob.Raw).To(MatchJSON(test_data.FrobModel.Raw))
})
It("does not duplicate frob events", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
It("marks header as checked for logs", func() {
err = frobRepository.Create(headerID, []frob.FrobModel{test_data.FrobModel})
Expect(err).NotTo(HaveOccurred())
frobRepository := frob.NewFrobRepository(db)
err = frobRepository.Create(headerID, test_data.FrobModel)
var headerChecked bool
err = db.Get(&headerChecked, `SELECT frob_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue())
})
It("does not duplicate frob events", func() {
err = frobRepository.Create(headerID, []frob.FrobModel{test_data.FrobModel})
Expect(err).NotTo(HaveOccurred())
err = frobRepository.Create(headerID, test_data.FrobModel)
err = frobRepository.Create(headerID, []frob.FrobModel{test_data.FrobModel})
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
})
It("removes frob if corresponding header is deleted", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
frobRepository := frob.NewFrobRepository(db)
err = frobRepository.Create(headerID, test_data.FrobModel)
err = frobRepository.Create(headerID, []frob.FrobModel{test_data.FrobModel})
Expect(err).NotTo(HaveOccurred())
_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
@ -90,8 +98,48 @@ var _ = Describe("Frob repository", func() {
})
})
Describe("MarkHeaderChecked", func() {
var (
db *postgres.DB
frobRepository frob.Repository
err error
headerID int64
)
BeforeEach(func() {
db = test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
frobRepository = frob.NewFrobRepository(db)
})
It("creates a row for a new headerID", func() {
err = frobRepository.MarkHeaderChecked(headerID)
Expect(err).NotTo(HaveOccurred())
var headerChecked bool
err = db.Get(&headerChecked, `SELECT frob_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue())
})
It("updates row when headerID already exists", func() {
_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID)
err = frobRepository.MarkHeaderChecked(headerID)
Expect(err).NotTo(HaveOccurred())
var headerChecked bool
err = db.Get(&headerChecked, `SELECT frob_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue())
})
})
Describe("MissingHeaders", func() {
It("returns headers with no associated frob event", func() {
It("returns headers that haven't been checked", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
@ -106,7 +154,7 @@ var _ = Describe("Frob repository", func() {
Expect(err).NotTo(HaveOccurred())
}
frobRepository := frob.NewFrobRepository(db)
err := frobRepository.Create(headerIDs[1], test_data.FrobModel)
err := frobRepository.MarkHeaderChecked(headerIDs[1])
Expect(err).NotTo(HaveOccurred())
headers, err := frobRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
@ -117,6 +165,33 @@ var _ = Describe("Frob repository", func() {
Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
})
It("only treats headers as checked if frob logs have been checked", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
startingBlockNumber := int64(1)
frobdBlockNumber := int64(2)
endingBlockNumber := int64(3)
blockNumbers := []int64{startingBlockNumber, frobdBlockNumber, endingBlockNumber, endingBlockNumber + 1}
var headerIDs []int64
for _, n := range blockNumbers {
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
headerIDs = append(headerIDs, headerID)
Expect(err).NotTo(HaveOccurred())
}
frobRepository := frob.NewFrobRepository(db)
_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
Expect(err).NotTo(HaveOccurred())
headers, err := frobRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
Expect(err).NotTo(HaveOccurred())
Expect(len(headers)).To(Equal(3))
Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(frobdBlockNumber)))
Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(frobdBlockNumber)))
Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(frobdBlockNumber)))
})
It("only returns headers associated with the current node", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
@ -134,7 +209,7 @@ var _ = Describe("Frob repository", func() {
}
frobRepository := frob.NewFrobRepository(db)
frobRepositoryTwo := frob.NewFrobRepository(dbTwo)
err := frobRepository.Create(headerIDs[0], test_data.FrobModel)
err := frobRepository.MarkHeaderChecked(headerIDs[0])
Expect(err).NotTo(HaveOccurred())
nodeOneMissingHeaders, err := frobRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])

View File

@ -59,20 +59,24 @@ func (transformer FrobTransformer) Execute() error {
if err != nil {
return err
}
for _, log := range matchingLogs {
entity, err := transformer.Converter.ToEntity(log.Address.Hex(), FrobConfig.ContractAbi, log)
if err != nil {
return err
}
model, err := transformer.Converter.ToModel(entity)
if err != nil {
return err
}
err = transformer.Repository.Create(header.Id, model)
if len(matchingLogs) < 1 {
err = transformer.Repository.MarkHeaderChecked(header.Id)
if err != nil {
return err
}
}
entities, err := transformer.Converter.ToEntities(FrobConfig.ContractAbi, matchingLogs)
if err != nil {
return err
}
models, err := transformer.Converter.ToModels(entities)
if err != nil {
return err
}
err = transformer.Repository.Create(header.Id, models)
if err != nil {
return err
}
}
return nil
}

View File

@ -96,6 +96,42 @@ var _ = Describe("Frob transformer", func() {
Expect(err).To(MatchError(fakes.FakeError))
})
It("marks header checked if no logs returned", func() {
mockConverter := &frob_mocks.MockFrobConverter{}
mockRepository := &frob_mocks.MockFrobRepository{}
headerID := int64(123)
mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}})
mockFetcher := &mocks.MockLogFetcher{}
transformer := frob.FrobTransformer{
Converter: mockConverter,
Fetcher: mockFetcher,
Repository: mockRepository,
}
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
mockRepository.AssertMarkHeaderCheckedCalledWith(headerID)
})
It("returns error if marking header checked returns err", func() {
mockConverter := &frob_mocks.MockFrobConverter{}
mockRepository := &frob_mocks.MockFrobRepository{}
mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}})
mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError)
mockFetcher := &mocks.MockLogFetcher{}
transformer := frob.FrobTransformer{
Converter: mockConverter,
Fetcher: mockFetcher,
Repository: mockRepository,
}
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("converts matching logs to entity", func() {
converter := &frob_mocks.MockFrobConverter{}
fetcher := &mocks.MockLogFetcher{}
@ -111,14 +147,13 @@ var _ = Describe("Frob transformer", func() {
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(converter.PassedContractAddress).To(Equal(test_data.EthFrobLog.Address.Hex()))
Expect(converter.PassedContractABI).To(Equal(frob.FrobConfig.ContractAbi))
Expect(converter.PassedLog).To(Equal(test_data.EthFrobLog))
Expect(converter.PassedLogs).To(Equal([]types.Log{test_data.EthFrobLog}))
})
It("returns error if converting to entity returns error", func() {
converter := &frob_mocks.MockFrobConverter{}
converter.SetToEntityError(fakes.FakeError)
converter.SetToEntitiesError(fakes.FakeError)
fetcher := &mocks.MockLogFetcher{}
fetcher.SetFetchedLogs([]types.Log{test_data.EthFrobLog})
repository := &frob_mocks.MockFrobRepository{}
@ -150,12 +185,12 @@ var _ = Describe("Frob transformer", func() {
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(converter.PassedEntity).To(Equal(test_data.FrobEntity))
Expect(converter.PassedEntities).To(Equal([]frob.FrobEntity{test_data.FrobEntity}))
})
It("returns error if converting to model returns error", func() {
converter := &frob_mocks.MockFrobConverter{}
converter.SetToModelError(fakes.FakeError)
converter.SetToModelsError(fakes.FakeError)
fetcher := &mocks.MockLogFetcher{}
fetcher.SetFetchedLogs([]types.Log{test_data.EthFrobLog})
repository := &frob_mocks.MockFrobRepository{}
@ -189,7 +224,7 @@ var _ = Describe("Frob transformer", func() {
Expect(err).NotTo(HaveOccurred())
Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id))
Expect(repository.PassedFrobModel).To(Equal(test_data.FrobModel))
Expect(repository.PassedModels).To(Equal([]frob.FrobModel{test_data.FrobModel}))
})
It("returns error if repository returns error for create", func() {

View File

@ -25,26 +25,34 @@ import (
)
type Converter interface {
ToModel(ethLog types.Log) (PitFileDebtCeilingModel, error)
ToModels(ethLogs []types.Log) ([]PitFileDebtCeilingModel, error)
}
type PitFileDebtCeilingConverter struct{}
func (PitFileDebtCeilingConverter) ToModel(ethLog types.Log) (PitFileDebtCeilingModel, error) {
err := verifyLog(ethLog)
if err != nil {
return PitFileDebtCeilingModel{}, err
}
what := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
data := big.NewInt(0).SetBytes(ethLog.Topics[3].Bytes()).String()
func (PitFileDebtCeilingConverter) ToModels(ethLogs []types.Log) ([]PitFileDebtCeilingModel, error) {
var models []PitFileDebtCeilingModel
for _, ethLog := range ethLogs {
err := verifyLog(ethLog)
if err != nil {
return nil, err
}
what := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
data := big.NewInt(0).SetBytes(ethLog.Topics[3].Bytes()).String()
raw, err := json.Marshal(ethLog)
return PitFileDebtCeilingModel{
What: what,
Data: data,
TransactionIndex: ethLog.TxIndex,
Raw: raw,
}, err
raw, err := json.Marshal(ethLog)
if err != nil {
return nil, err
}
model := PitFileDebtCeilingModel{
What: what,
Data: data,
TransactionIndex: ethLog.TxIndex,
Raw: raw,
}
models = append(models, model)
}
return models, nil
}
func verifyLog(log types.Log) error {

View File

@ -31,7 +31,7 @@ var _ = Describe("Pit file debt ceiling converter", func() {
Data: []byte{1, 1, 1, 1, 1},
}
_, err := converter.ToModel(badLog)
_, err := converter.ToModels([]types.Log{badLog})
Expect(err).To(HaveOccurred())
})
@ -42,7 +42,7 @@ var _ = Describe("Pit file debt ceiling converter", func() {
Topics: []common.Hash{{}, {}, {}, {}},
}
_, err := converter.ToModel(badLog)
_, err := converter.ToModels([]types.Log{badLog})
Expect(err).To(HaveOccurred())
})
@ -50,9 +50,10 @@ var _ = Describe("Pit file debt ceiling converter", func() {
It("converts a log to an model", func() {
converter := debt_ceiling.PitFileDebtCeilingConverter{}
model, err := converter.ToModel(test_data.EthPitFileDebtCeilingLog)
models, err := converter.ToModels([]types.Log{test_data.EthPitFileDebtCeilingLog})
Expect(err).NotTo(HaveOccurred())
Expect(model).To(Equal(test_data.PitFileDebtCeilingModel))
Expect(len(models)).To(Equal(1))
Expect(models[0]).To(Equal(test_data.PitFileDebtCeilingModel))
})
})

View File

@ -20,7 +20,8 @@ import (
)
type Repository interface {
Create(headerID int64, model PitFileDebtCeilingModel) error
Create(headerID int64, models []PitFileDebtCeilingModel) error
MarkHeaderChecked(headerID int64) error
MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
}
@ -34,12 +35,38 @@ func NewPitFileDebtCeilingRepository(db *postgres.DB) PitFileDebtCeilingReposito
}
}
func (repository PitFileDebtCeilingRepository) Create(headerID int64, model PitFileDebtCeilingModel) error {
_, err := repository.db.Exec(
`INSERT into maker.pit_file_debt_ceiling (header_id, what, data, tx_idx, raw_log)
func (repository PitFileDebtCeilingRepository) Create(headerID int64, models []PitFileDebtCeilingModel) error {
tx, err := repository.db.Begin()
if err != nil {
return err
}
for _, model := range models {
_, err = tx.Exec(
`INSERT into maker.pit_file_debt_ceiling (header_id, what, data, tx_idx, raw_log)
VALUES($1, $2, $3::NUMERIC, $4, $5)`,
headerID, model.What, model.Data, model.TransactionIndex, model.Raw,
)
headerID, model.What, model.Data, model.TransactionIndex, model.Raw,
)
if err != nil {
tx.Rollback()
return err
}
}
_, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, pit_file_debt_ceiling_checked)
VALUES ($1, $2)
ON CONFLICT (header_id) DO
UPDATE SET pit_file_debt_ceiling_checked = $2`, headerID, true)
if err != nil {
tx.Rollback()
return err
}
return tx.Commit()
}
func (repository PitFileDebtCeilingRepository) MarkHeaderChecked(headerID int64) error {
_, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, pit_file_debt_ceiling_checked)
VALUES ($1, $2)
ON CONFLICT (header_id) DO
UPDATE SET pit_file_debt_ceiling_checked = $2`, headerID, true)
return err
}
@ -48,8 +75,8 @@ func (repository PitFileDebtCeilingRepository) MissingHeaders(startingBlockNumbe
err := repository.db.Select(
&result,
`SELECT headers.id, headers.block_number FROM headers
LEFT JOIN maker.pit_file_debt_ceiling on headers.id = header_id
WHERE header_id ISNULL
LEFT JOIN checked_headers on headers.id = header_id
WHERE (header_id ISNULL OR pit_file_debt_ceiling_checked IS FALSE)
AND headers.block_number >= $1
AND headers.block_number <= $2
AND headers.eth_node_fingerprint = $3`,
@ -57,6 +84,5 @@ func (repository PitFileDebtCeilingRepository) MissingHeaders(startingBlockNumbe
endingBlockNumber,
repository.db.Node.ID,
)
return result, err
}

View File

@ -21,6 +21,7 @@ import (
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/debt_ceiling"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
@ -29,15 +30,24 @@ import (
var _ = Describe("Pit file debt ceiling repository", func() {
Describe("Create", func() {
It("adds a pit file debt ceiling event", func() {
db := test_config.NewTestDB(core.Node{})
var (
db *postgres.DB
pitFileDebtCeilingRepository debt_ceiling.Repository
err error
headerID int64
)
BeforeEach(func() {
db = test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
pitFileRepository := debt_ceiling.NewPitFileDebtCeilingRepository(db)
pitFileDebtCeilingRepository = debt_ceiling.NewPitFileDebtCeilingRepository(db)
})
err = pitFileRepository.Create(headerID, test_data.PitFileDebtCeilingModel)
It("adds a pit file debt ceiling event", func() {
err = pitFileDebtCeilingRepository.Create(headerID, []debt_ceiling.PitFileDebtCeilingModel{test_data.PitFileDebtCeilingModel})
Expect(err).NotTo(HaveOccurred())
var dbPitFile debt_ceiling.PitFileDebtCeilingModel
@ -49,30 +59,28 @@ var _ = Describe("Pit file debt ceiling repository", func() {
Expect(dbPitFile.Raw).To(MatchJSON(test_data.PitFileDebtCeilingModel.Raw))
})
It("does not duplicate pit file events", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
It("marks header as checked for logs", func() {
err = pitFileDebtCeilingRepository.Create(headerID, []debt_ceiling.PitFileDebtCeilingModel{test_data.PitFileDebtCeilingModel})
Expect(err).NotTo(HaveOccurred())
pitFileRepository := debt_ceiling.NewPitFileDebtCeilingRepository(db)
err = pitFileRepository.Create(headerID, test_data.PitFileDebtCeilingModel)
var headerChecked bool
err = db.Get(&headerChecked, `SELECT pit_file_debt_ceiling_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue())
})
It("does not duplicate pit file events", func() {
err = pitFileDebtCeilingRepository.Create(headerID, []debt_ceiling.PitFileDebtCeilingModel{test_data.PitFileDebtCeilingModel})
Expect(err).NotTo(HaveOccurred())
err = pitFileRepository.Create(headerID, test_data.PitFileDebtCeilingModel)
err = pitFileDebtCeilingRepository.Create(headerID, []debt_ceiling.PitFileDebtCeilingModel{test_data.PitFileDebtCeilingModel})
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
})
It("removes pit file if corresponding header is deleted", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
pitFileRepository := debt_ceiling.NewPitFileDebtCeilingRepository(db)
err = pitFileRepository.Create(headerID, test_data.PitFileDebtCeilingModel)
err = pitFileDebtCeilingRepository.Create(headerID, []debt_ceiling.PitFileDebtCeilingModel{test_data.PitFileDebtCeilingModel})
Expect(err).NotTo(HaveOccurred())
_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
@ -85,8 +93,48 @@ var _ = Describe("Pit file debt ceiling repository", func() {
})
})
Describe("MarkHeaderChecked", func() {
var (
db *postgres.DB
pitFileDebtCeilingRepository debt_ceiling.Repository
err error
headerID int64
)
BeforeEach(func() {
db = test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
pitFileDebtCeilingRepository = debt_ceiling.NewPitFileDebtCeilingRepository(db)
})
It("creates a row for a new headerID", func() {
err = pitFileDebtCeilingRepository.MarkHeaderChecked(headerID)
Expect(err).NotTo(HaveOccurred())
var headerChecked bool
err = db.Get(&headerChecked, `SELECT pit_file_debt_ceiling_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue())
})
It("updates row when headerID already exists", func() {
_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID)
err = pitFileDebtCeilingRepository.MarkHeaderChecked(headerID)
Expect(err).NotTo(HaveOccurred())
var headerChecked bool
err = db.Get(&headerChecked, `SELECT pit_file_debt_ceiling_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue())
})
})
Describe("MissingHeaders", func() {
It("returns headers with no associated pit file event", func() {
It("returns headers that haven't been checked", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
@ -101,7 +149,7 @@ var _ = Describe("Pit file debt ceiling repository", func() {
Expect(err).NotTo(HaveOccurred())
}
pitFileRepository := debt_ceiling.NewPitFileDebtCeilingRepository(db)
err := pitFileRepository.Create(headerIDs[1], test_data.PitFileDebtCeilingModel)
err := pitFileRepository.MarkHeaderChecked(headerIDs[1])
Expect(err).NotTo(HaveOccurred())
headers, err := pitFileRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
@ -112,6 +160,33 @@ var _ = Describe("Pit file debt ceiling repository", func() {
Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
})
It("only treats headers as checked if pit file debt ceiling logs have been checked", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
startingBlockNumber := int64(1)
pitFileDebtCeilingdBlockNumber := int64(2)
endingBlockNumber := int64(3)
blockNumbers := []int64{startingBlockNumber, pitFileDebtCeilingdBlockNumber, endingBlockNumber, endingBlockNumber + 1}
var headerIDs []int64
for _, n := range blockNumbers {
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
headerIDs = append(headerIDs, headerID)
Expect(err).NotTo(HaveOccurred())
}
pitFileDebtCeilingRepository := debt_ceiling.NewPitFileDebtCeilingRepository(db)
_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
Expect(err).NotTo(HaveOccurred())
headers, err := pitFileDebtCeilingRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
Expect(err).NotTo(HaveOccurred())
Expect(len(headers)).To(Equal(3))
Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(pitFileDebtCeilingdBlockNumber)))
Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(pitFileDebtCeilingdBlockNumber)))
Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(pitFileDebtCeilingdBlockNumber)))
})
It("only returns headers associated with the current node", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
@ -129,7 +204,7 @@ var _ = Describe("Pit file debt ceiling repository", func() {
}
pitFileRepository := debt_ceiling.NewPitFileDebtCeilingRepository(db)
pitFileRepositoryTwo := debt_ceiling.NewPitFileDebtCeilingRepository(dbTwo)
err := pitFileRepository.Create(headerIDs[0], test_data.PitFileDebtCeilingModel)
err := pitFileRepository.MarkHeaderChecked(headerIDs[0])
Expect(err).NotTo(HaveOccurred())
nodeOneMissingHeaders, err := pitFileRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])

View File

@ -60,16 +60,20 @@ func (transformer PitFileDebtCeilingTransformer) Execute() error {
if err != nil {
return err
}
for _, log := range matchingLogs {
model, err := transformer.Converter.ToModel(log)
if err != nil {
return err
}
err = transformer.Repository.Create(header.Id, model)
if len(matchingLogs) < 1 {
err = transformer.Repository.MarkHeaderChecked(header.Id)
if err != nil {
return err
}
}
model, err := transformer.Converter.ToModels(matchingLogs)
if err != nil {
return err
}
err = transformer.Repository.Create(header.Id, model)
if err != nil {
return err
}
}
return nil
}

View File

@ -97,6 +97,42 @@ var _ = Describe("Pit file debt ceiling transformer", func() {
Expect(err).To(MatchError(fakes.FakeError))
})
It("marks header checked if no logs returned", func() {
mockConverter := &debt_ceiling_mocks.MockPitFileDebtCeilingConverter{}
mockRepository := &debt_ceiling_mocks.MockPitFileDebtCeilingRepository{}
headerID := int64(123)
mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}})
mockFetcher := &mocks.MockLogFetcher{}
transformer := debt_ceiling.PitFileDebtCeilingTransformer{
Converter: mockConverter,
Fetcher: mockFetcher,
Repository: mockRepository,
}
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
mockRepository.AssertMarkHeaderCheckedCalledWith(headerID)
})
It("returns error if marking header checked returns err", func() {
mockConverter := &debt_ceiling_mocks.MockPitFileDebtCeilingConverter{}
mockRepository := &debt_ceiling_mocks.MockPitFileDebtCeilingRepository{}
mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}})
mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError)
mockFetcher := &mocks.MockLogFetcher{}
transformer := debt_ceiling.PitFileDebtCeilingTransformer{
Converter: mockConverter,
Fetcher: mockFetcher,
Repository: mockRepository,
}
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("converts matching logs", func() {
converter := &debt_ceiling_mocks.MockPitFileDebtCeilingConverter{}
fetcher := &mocks.MockLogFetcher{}
@ -112,7 +148,7 @@ var _ = Describe("Pit file debt ceiling transformer", func() {
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(converter.PassedLog).To(Equal(test_data.EthPitFileDebtCeilingLog))
Expect(converter.PassedLogs).To(Equal([]types.Log{test_data.EthPitFileDebtCeilingLog}))
})
It("returns error if converter returns error", func() {
@ -151,7 +187,7 @@ var _ = Describe("Pit file debt ceiling transformer", func() {
Expect(err).NotTo(HaveOccurred())
Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id))
Expect(repository.PassedModel).To(Equal(test_data.PitFileDebtCeilingModel))
Expect(repository.PassedModels).To(Equal([]debt_ceiling.PitFileDebtCeilingModel{test_data.PitFileDebtCeilingModel}))
})
It("returns error if repository returns error for create", func() {

View File

@ -25,29 +25,37 @@ import (
)
type Converter interface {
ToModel(ethLog types.Log) (PitFileIlkModel, error)
ToModels(ethLogs []types.Log) ([]PitFileIlkModel, error)
}
type PitFileIlkConverter struct{}
func (PitFileIlkConverter) ToModel(ethLog types.Log) (PitFileIlkModel, error) {
err := verifyLog(ethLog)
if err != nil {
return PitFileIlkModel{}, err
}
ilk := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
what := string(bytes.Trim(ethLog.Topics[3].Bytes(), "\x00"))
riskBytes := ethLog.Data[len(ethLog.Data)-shared.DataItemLength:]
risk := big.NewInt(0).SetBytes(riskBytes).String()
func (PitFileIlkConverter) ToModels(ethLogs []types.Log) ([]PitFileIlkModel, error) {
var models []PitFileIlkModel
for _, ethLog := range ethLogs {
err := verifyLog(ethLog)
if err != nil {
return nil, err
}
ilk := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
what := string(bytes.Trim(ethLog.Topics[3].Bytes(), "\x00"))
riskBytes := ethLog.Data[len(ethLog.Data)-shared.DataItemLength:]
risk := big.NewInt(0).SetBytes(riskBytes).String()
raw, err := json.Marshal(ethLog)
return PitFileIlkModel{
Ilk: ilk,
What: what,
Data: risk,
TransactionIndex: ethLog.TxIndex,
Raw: raw,
}, err
raw, err := json.Marshal(ethLog)
if err != nil {
return nil, err
}
model := PitFileIlkModel{
Ilk: ilk,
What: what,
Data: risk,
TransactionIndex: ethLog.TxIndex,
Raw: raw,
}
models = append(models, model)
}
return models, nil
}
func verifyLog(log types.Log) error {

View File

@ -31,7 +31,7 @@ var _ = Describe("Pit file ilk converter", func() {
Data: []byte{1, 1, 1, 1, 1},
}
_, err := converter.ToModel(badLog)
_, err := converter.ToModels([]types.Log{badLog})
Expect(err).To(HaveOccurred())
})
@ -42,7 +42,7 @@ var _ = Describe("Pit file ilk converter", func() {
Topics: []common.Hash{{}, {}, {}, {}},
}
_, err := converter.ToModel(badLog)
_, err := converter.ToModels([]types.Log{badLog})
Expect(err).To(HaveOccurred())
})
@ -50,9 +50,10 @@ var _ = Describe("Pit file ilk converter", func() {
It("converts a log to an model", func() {
converter := ilk.PitFileIlkConverter{}
model, err := converter.ToModel(test_data.EthPitFileIlkLog)
models, err := converter.ToModels([]types.Log{test_data.EthPitFileIlkLog})
Expect(err).NotTo(HaveOccurred())
Expect(model).To(Equal(test_data.PitFileIlkModel))
Expect(len(models)).To(Equal(1))
Expect(models[0]).To(Equal(test_data.PitFileIlkModel))
})
})

View File

@ -20,7 +20,8 @@ import (
)
type Repository interface {
Create(headerID int64, model PitFileIlkModel) error
Create(headerID int64, models []PitFileIlkModel) error
MarkHeaderChecked(headerID int64) error
MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
}
@ -34,12 +35,38 @@ func NewPitFileIlkRepository(db *postgres.DB) PitFileIlkRepository {
}
}
func (repository PitFileIlkRepository) Create(headerID int64, model PitFileIlkModel) error {
_, err := repository.db.Exec(
`INSERT into maker.pit_file_ilk (header_id, ilk, what, data, tx_idx, raw_log)
func (repository PitFileIlkRepository) Create(headerID int64, models []PitFileIlkModel) error {
tx, err := repository.db.Begin()
if err != nil {
return err
}
for _, model := range models {
_, err = tx.Exec(
`INSERT into maker.pit_file_ilk (header_id, ilk, what, data, tx_idx, raw_log)
VALUES($1, $2, $3, $4::NUMERIC, $5, $6)`,
headerID, model.Ilk, model.What, model.Data, model.TransactionIndex, model.Raw,
)
headerID, model.Ilk, model.What, model.Data, model.TransactionIndex, model.Raw,
)
if err != nil {
tx.Rollback()
return err
}
}
_, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, pit_file_ilk_checked)
VALUES ($1, $2)
ON CONFLICT (header_id) DO
UPDATE SET pit_file_ilk_checked = $2`, headerID, true)
if err != nil {
tx.Rollback()
return err
}
return tx.Commit()
}
func (repository PitFileIlkRepository) MarkHeaderChecked(headerID int64) error {
_, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, pit_file_ilk_checked)
VALUES ($1, $2)
ON CONFLICT (header_id) DO
UPDATE SET pit_file_ilk_checked = $2`, headerID, true)
return err
}
@ -48,8 +75,8 @@ func (repository PitFileIlkRepository) MissingHeaders(startingBlockNumber, endin
err := repository.db.Select(
&result,
`SELECT headers.id, headers.block_number FROM headers
LEFT JOIN maker.pit_file_ilk on headers.id = header_id
WHERE header_id ISNULL
LEFT JOIN checked_headers on headers.id = header_id
WHERE (header_id ISNULL OR pit_file_ilk_checked IS FALSE)
AND headers.block_number >= $1
AND headers.block_number <= $2
AND headers.eth_node_fingerprint = $3`,
@ -57,6 +84,5 @@ func (repository PitFileIlkRepository) MissingHeaders(startingBlockNumber, endin
endingBlockNumber,
repository.db.Node.ID,
)
return result, err
}

View File

@ -21,6 +21,7 @@ import (
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/ilk"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
@ -29,15 +30,24 @@ import (
var _ = Describe("Pit file ilk repository", func() {
Describe("Create", func() {
It("adds a pit file ilk event", func() {
db := test_config.NewTestDB(core.Node{})
var (
db *postgres.DB
pitFileIlkRepository ilk.Repository
err error
headerID int64
)
BeforeEach(func() {
db = test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
pitFileRepository := ilk.NewPitFileIlkRepository(db)
pitFileIlkRepository = ilk.NewPitFileIlkRepository(db)
})
err = pitFileRepository.Create(headerID, test_data.PitFileIlkModel)
It("adds a pit file ilk event", func() {
err = pitFileIlkRepository.Create(headerID, []ilk.PitFileIlkModel{test_data.PitFileIlkModel})
Expect(err).NotTo(HaveOccurred())
var dbPitFile ilk.PitFileIlkModel
@ -50,30 +60,28 @@ var _ = Describe("Pit file ilk repository", func() {
Expect(dbPitFile.Raw).To(MatchJSON(test_data.PitFileIlkModel.Raw))
})
It("does not duplicate pit file ilk events", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
It("marks header as checked for logs", func() {
err = pitFileIlkRepository.Create(headerID, []ilk.PitFileIlkModel{test_data.PitFileIlkModel})
Expect(err).NotTo(HaveOccurred())
pitFileRepository := ilk.NewPitFileIlkRepository(db)
err = pitFileRepository.Create(headerID, test_data.PitFileIlkModel)
var headerChecked bool
err = db.Get(&headerChecked, `SELECT pit_file_ilk_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue())
})
It("does not duplicate pit file ilk events", func() {
err = pitFileIlkRepository.Create(headerID, []ilk.PitFileIlkModel{test_data.PitFileIlkModel})
Expect(err).NotTo(HaveOccurred())
err = pitFileRepository.Create(headerID, test_data.PitFileIlkModel)
err = pitFileIlkRepository.Create(headerID, []ilk.PitFileIlkModel{test_data.PitFileIlkModel})
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
})
It("removes pit file ilk if corresponding header is deleted", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
pitFileRepository := ilk.NewPitFileIlkRepository(db)
err = pitFileRepository.Create(headerID, test_data.PitFileIlkModel)
err = pitFileIlkRepository.Create(headerID, []ilk.PitFileIlkModel{test_data.PitFileIlkModel})
Expect(err).NotTo(HaveOccurred())
_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
@ -86,8 +94,48 @@ var _ = Describe("Pit file ilk repository", func() {
})
})
Describe("MarkHeaderChecked", func() {
var (
db *postgres.DB
pitFileIlkRepository ilk.Repository
err error
headerID int64
)
BeforeEach(func() {
db = test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
pitFileIlkRepository = ilk.NewPitFileIlkRepository(db)
})
It("creates a row for a new headerID", func() {
err = pitFileIlkRepository.MarkHeaderChecked(headerID)
Expect(err).NotTo(HaveOccurred())
var headerChecked bool
err = db.Get(&headerChecked, `SELECT pit_file_ilk_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue())
})
It("updates row when headerID already exists", func() {
_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID)
err = pitFileIlkRepository.MarkHeaderChecked(headerID)
Expect(err).NotTo(HaveOccurred())
var headerChecked bool
err = db.Get(&headerChecked, `SELECT pit_file_ilk_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue())
})
})
Describe("MissingHeaders", func() {
It("returns headers with no associated pit file ilk event", func() {
It("returns headers that haven't been checked", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
@ -102,7 +150,7 @@ var _ = Describe("Pit file ilk repository", func() {
Expect(err).NotTo(HaveOccurred())
}
pitFileRepository := ilk.NewPitFileIlkRepository(db)
err := pitFileRepository.Create(headerIDs[1], test_data.PitFileIlkModel)
err := pitFileRepository.MarkHeaderChecked(headerIDs[1])
Expect(err).NotTo(HaveOccurred())
headers, err := pitFileRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
@ -113,6 +161,33 @@ var _ = Describe("Pit file ilk repository", func() {
Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
})
It("only treats headers as checked if pit file ilk logs have been checked", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
startingBlockNumber := int64(1)
pitFileIlkdBlockNumber := int64(2)
endingBlockNumber := int64(3)
blockNumbers := []int64{startingBlockNumber, pitFileIlkdBlockNumber, endingBlockNumber, endingBlockNumber + 1}
var headerIDs []int64
for _, n := range blockNumbers {
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
headerIDs = append(headerIDs, headerID)
Expect(err).NotTo(HaveOccurred())
}
pitFileIlkRepository := ilk.NewPitFileIlkRepository(db)
_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
Expect(err).NotTo(HaveOccurred())
headers, err := pitFileIlkRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
Expect(err).NotTo(HaveOccurred())
Expect(len(headers)).To(Equal(3))
Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(pitFileIlkdBlockNumber)))
Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(pitFileIlkdBlockNumber)))
Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(pitFileIlkdBlockNumber)))
})
It("only returns headers associated with the current node", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
@ -130,7 +205,7 @@ var _ = Describe("Pit file ilk repository", func() {
}
pitFileRepository := ilk.NewPitFileIlkRepository(db)
pitFileRepositoryTwo := ilk.NewPitFileIlkRepository(dbTwo)
err := pitFileRepository.Create(headerIDs[0], test_data.PitFileIlkModel)
err := pitFileRepository.MarkHeaderChecked(headerIDs[0])
Expect(err).NotTo(HaveOccurred())
nodeOneMissingHeaders, err := pitFileRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])

View File

@ -60,16 +60,20 @@ func (transformer PitFileIlkTransformer) Execute() error {
if err != nil {
return err
}
for _, log := range matchingLogs {
model, err := transformer.Converter.ToModel(log)
if err != nil {
return err
}
err = transformer.Repository.Create(header.Id, model)
if len(matchingLogs) < 1 {
err = transformer.Repository.MarkHeaderChecked(header.Id)
if err != nil {
return err
}
}
models, err := transformer.Converter.ToModels(matchingLogs)
if err != nil {
return err
}
err = transformer.Repository.Create(header.Id, models)
if err != nil {
return err
}
}
return nil
}

View File

@ -97,6 +97,42 @@ var _ = Describe("Pit file ilk transformer", func() {
Expect(err).To(MatchError(fakes.FakeError))
})
It("marks header checked if no logs returned", func() {
mockConverter := &pit_file_ilk_mocks.MockPitFileIlkConverter{}
mockRepository := &pit_file_ilk_mocks.MockPitFileIlkRepository{}
headerID := int64(123)
mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}})
mockFetcher := &mocks.MockLogFetcher{}
transformer := ilk.PitFileIlkTransformer{
Converter: mockConverter,
Fetcher: mockFetcher,
Repository: mockRepository,
}
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
mockRepository.AssertMarkHeaderCheckedCalledWith(headerID)
})
It("returns error if marking header checked returns err", func() {
mockConverter := &pit_file_ilk_mocks.MockPitFileIlkConverter{}
mockRepository := &pit_file_ilk_mocks.MockPitFileIlkRepository{}
mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}})
mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError)
mockFetcher := &mocks.MockLogFetcher{}
transformer := ilk.PitFileIlkTransformer{
Converter: mockConverter,
Fetcher: mockFetcher,
Repository: mockRepository,
}
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("converts matching logs", func() {
converter := &pit_file_ilk_mocks.MockPitFileIlkConverter{}
fetcher := &mocks.MockLogFetcher{}
@ -112,7 +148,7 @@ var _ = Describe("Pit file ilk transformer", func() {
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(converter.PassedLog).To(Equal(test_data.EthPitFileIlkLog))
Expect(converter.PassedLogs).To(Equal([]types.Log{test_data.EthPitFileIlkLog}))
})
It("returns error if converter returns error", func() {
@ -151,7 +187,7 @@ var _ = Describe("Pit file ilk transformer", func() {
Expect(err).NotTo(HaveOccurred())
Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id))
Expect(repository.PassedModel).To(Equal(test_data.PitFileIlkModel))
Expect(repository.PassedModels).To(Equal([]ilk.PitFileIlkModel{test_data.PitFileIlkModel}))
})
It("returns error if repository returns error for create", func() {

View File

@ -24,26 +24,34 @@ import (
)
type Converter interface {
ToModel(ethLog types.Log) (PitFileStabilityFeeModel, error)
ToModels(ethLogs []types.Log) ([]PitFileStabilityFeeModel, error)
}
type PitFileStabilityFeeConverter struct{}
func (PitFileStabilityFeeConverter) ToModel(ethLog types.Log) (PitFileStabilityFeeModel, error) {
err := verifyLog(ethLog)
if err != nil {
return PitFileStabilityFeeModel{}, err
}
what := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
data := common.HexToAddress(ethLog.Topics[3].String()).Hex()
func (PitFileStabilityFeeConverter) ToModels(ethLogs []types.Log) ([]PitFileStabilityFeeModel, error) {
var models []PitFileStabilityFeeModel
for _, ethLog := range ethLogs {
err := verifyLog(ethLog)
if err != nil {
return nil, err
}
what := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
data := common.HexToAddress(ethLog.Topics[3].String()).Hex()
raw, err := json.Marshal(ethLog)
return PitFileStabilityFeeModel{
What: what,
Data: data,
TransactionIndex: ethLog.TxIndex,
Raw: raw,
}, err
raw, err := json.Marshal(ethLog)
if err != nil {
return nil, err
}
model := PitFileStabilityFeeModel{
What: what,
Data: data,
TransactionIndex: ethLog.TxIndex,
Raw: raw,
}
models = append(models, model)
}
return models, nil
}
func verifyLog(log types.Log) error {

View File

@ -28,7 +28,7 @@ var _ = Describe("Pit file stability fee converter", func() {
converter := stability_fee.PitFileStabilityFeeConverter{}
badLog := types.Log{}
_, err := converter.ToModel(badLog)
_, err := converter.ToModels([]types.Log{badLog})
Expect(err).To(HaveOccurred())
})
@ -36,9 +36,10 @@ var _ = Describe("Pit file stability fee converter", func() {
It("converts a log to an model", func() {
converter := stability_fee.PitFileStabilityFeeConverter{}
model, err := converter.ToModel(test_data.EthPitFileStabilityFeeLog)
models, err := converter.ToModels([]types.Log{test_data.EthPitFileStabilityFeeLog})
Expect(err).NotTo(HaveOccurred())
Expect(model).To(Equal(test_data.PitFileStabilityFeeModel))
Expect(len(models)).To(Equal(1))
Expect(models[0]).To(Equal(test_data.PitFileStabilityFeeModel))
})
})

View File

@ -20,7 +20,8 @@ import (
)
type Repository interface {
Create(headerID int64, model PitFileStabilityFeeModel) error
Create(headerID int64, models []PitFileStabilityFeeModel) error
MarkHeaderChecked(headerID int64) error
MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
}
@ -34,12 +35,38 @@ func NewPitFileStabilityFeeRepository(db *postgres.DB) PitFileStabilityFeeReposi
}
}
func (repository PitFileStabilityFeeRepository) Create(headerID int64, model PitFileStabilityFeeModel) error {
_, err := repository.db.Exec(
`INSERT into maker.pit_file_stability_fee (header_id, what, data, tx_idx, raw_log)
func (repository PitFileStabilityFeeRepository) Create(headerID int64, models []PitFileStabilityFeeModel) error {
tx, err := repository.db.Begin()
if err != nil {
return err
}
for _, model := range models {
_, err = tx.Exec(
`INSERT into maker.pit_file_stability_fee (header_id, what, data, tx_idx, raw_log)
VALUES($1, $2, $3, $4, $5)`,
headerID, model.What, model.Data, model.TransactionIndex, model.Raw,
)
headerID, model.What, model.Data, model.TransactionIndex, model.Raw,
)
if err != nil {
tx.Rollback()
return err
}
}
_, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, pit_file_stability_fee_checked)
VALUES ($1, $2)
ON CONFLICT (header_id) DO
UPDATE SET pit_file_stability_fee_checked = $2`, headerID, true)
if err != nil {
tx.Rollback()
return err
}
return tx.Commit()
}
func (repository PitFileStabilityFeeRepository) MarkHeaderChecked(headerID int64) error {
_, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, pit_file_stability_fee_checked)
VALUES ($1, $2)
ON CONFLICT (header_id) DO
UPDATE SET pit_file_stability_fee_checked = $2`, headerID, true)
return err
}
@ -48,8 +75,8 @@ func (repository PitFileStabilityFeeRepository) MissingHeaders(startingBlockNumb
err := repository.db.Select(
&result,
`SELECT headers.id, headers.block_number FROM headers
LEFT JOIN maker.pit_file_stability_fee on headers.id = header_id
WHERE header_id ISNULL
LEFT JOIN checked_headers on headers.id = header_id
WHERE (header_id ISNULL OR pit_file_stability_fee_checked IS FALSE)
AND headers.block_number >= $1
AND headers.block_number <= $2
AND headers.eth_node_fingerprint = $3`,
@ -57,6 +84,5 @@ func (repository PitFileStabilityFeeRepository) MissingHeaders(startingBlockNumb
endingBlockNumber,
repository.db.Node.ID,
)
return result, err
}

View File

@ -21,6 +21,7 @@ import (
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/stability_fee"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
@ -29,15 +30,24 @@ import (
var _ = Describe("Pit file stability fee repository", func() {
Describe("Create", func() {
It("adds a pit file stability fee event", func() {
db := test_config.NewTestDB(core.Node{})
var (
db *postgres.DB
pitFileStabilityFeeRepository stability_fee.Repository
err error
headerID int64
)
BeforeEach(func() {
db = test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
pitFileRepository := stability_fee.NewPitFileStabilityFeeRepository(db)
pitFileStabilityFeeRepository = stability_fee.NewPitFileStabilityFeeRepository(db)
})
err = pitFileRepository.Create(headerID, test_data.PitFileStabilityFeeModel)
It("adds a pit file stability fee event", func() {
err = pitFileStabilityFeeRepository.Create(headerID, []stability_fee.PitFileStabilityFeeModel{test_data.PitFileStabilityFeeModel})
Expect(err).NotTo(HaveOccurred())
var dbPitFile stability_fee.PitFileStabilityFeeModel
@ -49,30 +59,28 @@ var _ = Describe("Pit file stability fee repository", func() {
Expect(dbPitFile.Raw).To(MatchJSON(test_data.PitFileStabilityFeeModel.Raw))
})
It("does not duplicate pit file events", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
It("marks header as checked for logs", func() {
err = pitFileStabilityFeeRepository.Create(headerID, []stability_fee.PitFileStabilityFeeModel{test_data.PitFileStabilityFeeModel})
Expect(err).NotTo(HaveOccurred())
pitFileRepository := stability_fee.NewPitFileStabilityFeeRepository(db)
err = pitFileRepository.Create(headerID, test_data.PitFileStabilityFeeModel)
var headerChecked bool
err = db.Get(&headerChecked, `SELECT pit_file_stability_fee_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue())
})
It("does not duplicate pit file events", func() {
err = pitFileStabilityFeeRepository.Create(headerID, []stability_fee.PitFileStabilityFeeModel{test_data.PitFileStabilityFeeModel})
Expect(err).NotTo(HaveOccurred())
err = pitFileRepository.Create(headerID, test_data.PitFileStabilityFeeModel)
err = pitFileStabilityFeeRepository.Create(headerID, []stability_fee.PitFileStabilityFeeModel{test_data.PitFileStabilityFeeModel})
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
})
It("removes pit file if corresponding header is deleted", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
pitFileRepository := stability_fee.NewPitFileStabilityFeeRepository(db)
err = pitFileRepository.Create(headerID, test_data.PitFileStabilityFeeModel)
err = pitFileStabilityFeeRepository.Create(headerID, []stability_fee.PitFileStabilityFeeModel{test_data.PitFileStabilityFeeModel})
Expect(err).NotTo(HaveOccurred())
_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
@ -85,8 +93,48 @@ var _ = Describe("Pit file stability fee repository", func() {
})
})
Describe("MarkHeaderChecked", func() {
var (
db *postgres.DB
pitFileStabilityFeeRepository stability_fee.Repository
err error
headerID int64
)
BeforeEach(func() {
db = test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
pitFileStabilityFeeRepository = stability_fee.NewPitFileStabilityFeeRepository(db)
})
It("creates a row for a new headerID", func() {
err = pitFileStabilityFeeRepository.MarkHeaderChecked(headerID)
Expect(err).NotTo(HaveOccurred())
var headerChecked bool
err = db.Get(&headerChecked, `SELECT pit_file_stability_fee_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue())
})
It("updates row when headerID already exists", func() {
_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID)
err = pitFileStabilityFeeRepository.MarkHeaderChecked(headerID)
Expect(err).NotTo(HaveOccurred())
var headerChecked bool
err = db.Get(&headerChecked, `SELECT pit_file_stability_fee_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue())
})
})
Describe("MissingHeaders", func() {
It("returns headers with no associated pit file event", func() {
It("returns headers that haven't been checked", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
@ -101,7 +149,7 @@ var _ = Describe("Pit file stability fee repository", func() {
Expect(err).NotTo(HaveOccurred())
}
pitFileRepository := stability_fee.NewPitFileStabilityFeeRepository(db)
err := pitFileRepository.Create(headerIDs[1], test_data.PitFileStabilityFeeModel)
err := pitFileRepository.MarkHeaderChecked(headerIDs[1])
Expect(err).NotTo(HaveOccurred())
headers, err := pitFileRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
@ -112,6 +160,33 @@ var _ = Describe("Pit file stability fee repository", func() {
Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
})
It("only treats headers as checked if pit file stability fee logs have been checked", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
startingBlockNumber := int64(1)
pitFileStabilityFeedBlockNumber := int64(2)
endingBlockNumber := int64(3)
blockNumbers := []int64{startingBlockNumber, pitFileStabilityFeedBlockNumber, endingBlockNumber, endingBlockNumber + 1}
var headerIDs []int64
for _, n := range blockNumbers {
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
headerIDs = append(headerIDs, headerID)
Expect(err).NotTo(HaveOccurred())
}
pitFileStabilityFeeRepository := stability_fee.NewPitFileStabilityFeeRepository(db)
_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
Expect(err).NotTo(HaveOccurred())
headers, err := pitFileStabilityFeeRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
Expect(err).NotTo(HaveOccurred())
Expect(len(headers)).To(Equal(3))
Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(pitFileStabilityFeedBlockNumber)))
Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(pitFileStabilityFeedBlockNumber)))
Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(pitFileStabilityFeedBlockNumber)))
})
It("only returns headers associated with the current node", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
@ -129,7 +204,7 @@ var _ = Describe("Pit file stability fee repository", func() {
}
pitFileRepository := stability_fee.NewPitFileStabilityFeeRepository(db)
pitFileRepositoryTwo := stability_fee.NewPitFileStabilityFeeRepository(dbTwo)
err := pitFileRepository.Create(headerIDs[0], test_data.PitFileStabilityFeeModel)
err := pitFileRepository.MarkHeaderChecked(headerIDs[0])
Expect(err).NotTo(HaveOccurred())
nodeOneMissingHeaders, err := pitFileRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])

View File

@ -60,16 +60,20 @@ func (transformer PitFileStabilityFeeTransformer) Execute() error {
if err != nil {
return err
}
for _, log := range matchingLogs {
model, err := transformer.Converter.ToModel(log)
if err != nil {
return err
}
err = transformer.Repository.Create(header.Id, model)
if len(matchingLogs) < 1 {
err = transformer.Repository.MarkHeaderChecked(header.Id)
if err != nil {
return err
}
}
models, err := transformer.Converter.ToModels(matchingLogs)
if err != nil {
return err
}
err = transformer.Repository.Create(header.Id, models)
if err != nil {
return err
}
}
return nil
}

View File

@ -97,6 +97,42 @@ var _ = Describe("Pit file stability fee transformer", func() {
Expect(err).To(MatchError(fakes.FakeError))
})
It("marks header checked if no logs returned", func() {
mockConverter := &stability_fee_mocks.MockPitFileStabilityFeeConverter{}
mockRepository := &stability_fee_mocks.MockPitFileStabilityFeeRepository{}
headerID := int64(123)
mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}})
mockFetcher := &mocks.MockLogFetcher{}
transformer := stability_fee.PitFileStabilityFeeTransformer{
Converter: mockConverter,
Fetcher: mockFetcher,
Repository: mockRepository,
}
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
mockRepository.AssertMarkHeaderCheckedCalledWith(headerID)
})
It("returns error if marking header checked returns err", func() {
mockConverter := &stability_fee_mocks.MockPitFileStabilityFeeConverter{}
mockRepository := &stability_fee_mocks.MockPitFileStabilityFeeRepository{}
mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}})
mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError)
mockFetcher := &mocks.MockLogFetcher{}
transformer := stability_fee.PitFileStabilityFeeTransformer{
Converter: mockConverter,
Fetcher: mockFetcher,
Repository: mockRepository,
}
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("converts matching logs", func() {
converter := &stability_fee_mocks.MockPitFileStabilityFeeConverter{}
fetcher := &mocks.MockLogFetcher{}
@ -112,7 +148,7 @@ var _ = Describe("Pit file stability fee transformer", func() {
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(converter.PassedLog).To(Equal(test_data.EthPitFileStabilityFeeLog))
Expect(converter.PassedLogs).To(Equal([]types.Log{test_data.EthPitFileStabilityFeeLog}))
})
It("returns error if converter returns error", func() {
@ -151,7 +187,7 @@ var _ = Describe("Pit file stability fee transformer", func() {
Expect(err).NotTo(HaveOccurred())
Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id))
Expect(repository.PassedModel).To(Equal(test_data.PitFileStabilityFeeModel))
Expect(repository.PassedModels).To(Equal([]stability_fee.PitFileStabilityFeeModel{test_data.PitFileStabilityFeeModel}))
})
It("returns error if repository returns error for create", func() {

View File

@ -22,24 +22,23 @@ import (
)
type MockBiteConverter struct {
ConverterContract string
ConverterAbi string
LogsToConvert []types.Log
EntitiesToConvert []BiteEntity
ConverterError error
}
func (mbc *MockBiteConverter) ToEntity(contractAddress string, contractAbi string, ethLog types.Log) (BiteEntity, error) {
mbc.ConverterContract = contractAddress
func (mbc *MockBiteConverter) ToEntities(contractAbi string, ethLogs []types.Log) ([]BiteEntity, error) {
mbc.ConverterAbi = contractAbi
mbc.LogsToConvert = append(mbc.LogsToConvert, ethLog)
return test_data.BiteEntity, mbc.ConverterError
mbc.LogsToConvert = append(mbc.LogsToConvert, ethLogs...)
return []BiteEntity{test_data.BiteEntity}, mbc.ConverterError
}
func (mbc *MockBiteConverter) ToModel(entity BiteEntity) (BiteModel, error) {
mbc.EntitiesToConvert = append(mbc.EntitiesToConvert, entity)
return test_data.BiteModel, mbc.ConverterError
func (mbc *MockBiteConverter) ToModels(entities []BiteEntity) ([]BiteModel, error) {
mbc.EntitiesToConvert = append(mbc.EntitiesToConvert, entities...)
return []BiteModel{test_data.BiteModel}, mbc.ConverterError
}
func (mbc *MockBiteConverter) SetConverterError(err error) {
mbc.ConverterError = err
}

View File

@ -15,19 +15,23 @@
package bite
import (
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/transformers/bite"
)
type MockBiteRepository struct {
createError error
PassedEndingBlockNumber int64
PassedBiteModel bite.BiteModel
PassedHeaderID int64
PassedStartingBlockNumber int64
PassedTransactionIndex uint
missingHeaders []core.Header
missingHeadersErr error
createError error
PassedEndingBlockNumber int64
PassedBiteModels []bite.BiteModel
PassedHeaderID int64
PassedStartingBlockNumber int64
PassedTransactionIndex uint
markHeaderCheckedErr error
markHeaderCheckedPassedHeaderID int64
missingHeaders []core.Header
missingHeadersErr error
}
func (repository *MockBiteRepository) SetCreateError(err error) {
@ -42,14 +46,27 @@ func (repository *MockBiteRepository) SetMissingHeaders(headers []core.Header) {
repository.missingHeaders = headers
}
func (repository *MockBiteRepository) Create(headerID int64, model bite.BiteModel) error {
func (repository *MockBiteRepository) Create(headerID int64, models []bite.BiteModel) error {
repository.PassedHeaderID = headerID
repository.PassedBiteModel = model
repository.PassedBiteModels = models
return repository.createError
}
func (repository *MockBiteRepository) MarkHeaderChecked(headerID int64) error {
repository.markHeaderCheckedPassedHeaderID = headerID
return repository.markHeaderCheckedErr
}
func (repository *MockBiteRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
repository.PassedStartingBlockNumber = startingBlockNumber
repository.PassedEndingBlockNumber = endingBlockNumber
return repository.missingHeaders, repository.missingHeadersErr
}
func (repository *MockBiteRepository) SetMarkHeaderCheckedErr(e error) {
repository.markHeaderCheckedErr = e
}
func (repository *MockBiteRepository) AssertMarkHeaderCheckedCalledWith(i int64) {
Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i))
}

View File

@ -21,13 +21,13 @@ import (
)
type MockDripDripConverter struct {
Err error
PassedLog types.Log
Err error
PassedLogs []types.Log
}
func (converter *MockDripDripConverter) ToModel(ethLog types.Log) (drip_drip.DripDripModel, error) {
converter.PassedLog = ethLog
return test_data.DripDripModel, converter.Err
func (converter *MockDripDripConverter) ToModels(ethLogs []types.Log) ([]drip_drip.DripDripModel, error) {
converter.PassedLogs = ethLogs
return []drip_drip.DripDripModel{test_data.DripDripModel}, converter.Err
}
func (converter *MockDripDripConverter) SetConverterError(e error) {

View File

@ -15,32 +15,45 @@
package drip_drip
import (
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/transformers/drip_drip"
)
type MockDripDripRepository struct {
createErr error
missingHeaders []core.Header
missingHeadersErr error
PassedStartingBlockNumber int64
PassedEndingBlockNumber int64
PassedHeaderID int64
PassedModel drip_drip.DripDripModel
createErr error
markHeaderCheckedErr error
markHeaderCheckedPassedHeaderID int64
missingHeaders []core.Header
missingHeadersErr error
PassedStartingBlockNumber int64
PassedEndingBlockNumber int64
PassedHeaderID int64
PassedModels []drip_drip.DripDripModel
}
func (repository *MockDripDripRepository) Create(headerID int64, model drip_drip.DripDripModel) error {
func (repository *MockDripDripRepository) Create(headerID int64, models []drip_drip.DripDripModel) error {
repository.PassedHeaderID = headerID
repository.PassedModel = model
repository.PassedModels = models
return repository.createErr
}
func (repository *MockDripDripRepository) MarkHeaderChecked(headerID int64) error {
repository.markHeaderCheckedPassedHeaderID = headerID
return repository.markHeaderCheckedErr
}
func (repository *MockDripDripRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
repository.PassedStartingBlockNumber = startingBlockNumber
repository.PassedEndingBlockNumber = endingBlockNumber
return repository.missingHeaders, repository.missingHeadersErr
}
func (repository *MockDripDripRepository) SetMarkHeaderCheckedErr(err error) {
repository.markHeaderCheckedErr = err
}
func (repository *MockDripDripRepository) SetMissingHeadersErr(e error) {
repository.missingHeadersErr = e
}
@ -52,3 +65,6 @@ func (repository *MockDripDripRepository) SetMissingHeaders(headers []core.Heade
func (repository *MockDripDripRepository) SetCreateError(e error) {
repository.createErr = e
}
func (repository *MockDripDripRepository) AssertMarkHeaderCheckedCalledWith(i int64) {
Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i))
}

View File

@ -22,12 +22,12 @@ import (
type MockDripFileIlkConverter struct {
converterErr error
PassedLog types.Log
PassedLogs []types.Log
}
func (converter *MockDripFileIlkConverter) ToModel(ethLog types.Log) (ilk.DripFileIlkModel, error) {
converter.PassedLog = ethLog
return test_data.DripFileIlkModel, converter.converterErr
func (converter *MockDripFileIlkConverter) ToModels(ethLogs []types.Log) ([]ilk.DripFileIlkModel, error) {
converter.PassedLogs = ethLogs
return []ilk.DripFileIlkModel{test_data.DripFileIlkModel}, converter.converterErr
}
func (converter *MockDripFileIlkConverter) SetConverterError(e error) {

View File

@ -15,32 +15,45 @@
package ilk
import (
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/ilk"
)
type MockDripFileIlkRepository struct {
createErr error
missingHeaders []core.Header
missingHeadersErr error
PassedStartingBlockNumber int64
PassedEndingBlockNumber int64
PassedHeaderID int64
PassedModel ilk.DripFileIlkModel
createErr error
markHeaderCheckedErr error
markHeaderCheckedPassedHeaderID int64
missingHeaders []core.Header
missingHeadersErr error
PassedStartingBlockNumber int64
PassedEndingBlockNumber int64
PassedHeaderID int64
PassedModels []ilk.DripFileIlkModel
}
func (repository *MockDripFileIlkRepository) Create(headerID int64, model ilk.DripFileIlkModel) error {
func (repository *MockDripFileIlkRepository) Create(headerID int64, models []ilk.DripFileIlkModel) error {
repository.PassedHeaderID = headerID
repository.PassedModel = model
repository.PassedModels = models
return repository.createErr
}
func (repository *MockDripFileIlkRepository) MarkHeaderChecked(headerID int64) error {
repository.markHeaderCheckedPassedHeaderID = headerID
return repository.markHeaderCheckedErr
}
func (repository *MockDripFileIlkRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
repository.PassedStartingBlockNumber = startingBlockNumber
repository.PassedEndingBlockNumber = endingBlockNumber
return repository.missingHeaders, repository.missingHeadersErr
}
func (repository *MockDripFileIlkRepository) SetMarkHeaderCheckedErr(e error) {
repository.markHeaderCheckedErr = e
}
func (repository *MockDripFileIlkRepository) SetMissingHeadersErr(e error) {
repository.missingHeadersErr = e
}
@ -52,3 +65,6 @@ func (repository *MockDripFileIlkRepository) SetMissingHeaders(headers []core.He
func (repository *MockDripFileIlkRepository) SetCreateError(e error) {
repository.createErr = e
}
func (repository *MockDripFileIlkRepository) AssertMarkHeaderCheckedCalledWith(i int64) {
Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i))
}

View File

@ -22,12 +22,12 @@ import (
type MockDripFileRepoConverter struct {
converterErr error
PassedLog types.Log
PassedLogs []types.Log
}
func (converter *MockDripFileRepoConverter) ToModel(ethLog types.Log) (repo.DripFileRepoModel, error) {
converter.PassedLog = ethLog
return test_data.DripFileRepoModel, converter.converterErr
func (converter *MockDripFileRepoConverter) ToModels(ethLogs []types.Log) ([]repo.DripFileRepoModel, error) {
converter.PassedLogs = ethLogs
return []repo.DripFileRepoModel{test_data.DripFileRepoModel}, converter.converterErr
}
func (converter *MockDripFileRepoConverter) SetConverterError(e error) {

View File

@ -15,32 +15,45 @@
package repo
import (
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/repo"
)
type MockDripFileRepoRepository struct {
createErr error
missingHeaders []core.Header
missingHeadersErr error
PassedStartingBlockNumber int64
PassedEndingBlockNumber int64
PassedHeaderID int64
PassedModel repo.DripFileRepoModel
createErr error
markHeaderCheckedErr error
markHeaderCheckedPassedHeaderID int64
missingHeaders []core.Header
missingHeadersErr error
PassedStartingBlockNumber int64
PassedEndingBlockNumber int64
PassedHeaderID int64
PassedModels []repo.DripFileRepoModel
}
func (repository *MockDripFileRepoRepository) Create(headerID int64, model repo.DripFileRepoModel) error {
func (repository *MockDripFileRepoRepository) Create(headerID int64, models []repo.DripFileRepoModel) error {
repository.PassedHeaderID = headerID
repository.PassedModel = model
repository.PassedModels = models
return repository.createErr
}
func (repository *MockDripFileRepoRepository) MarkHeaderChecked(headerID int64) error {
repository.markHeaderCheckedPassedHeaderID = headerID
return repository.markHeaderCheckedErr
}
func (repository *MockDripFileRepoRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
repository.PassedStartingBlockNumber = startingBlockNumber
repository.PassedEndingBlockNumber = endingBlockNumber
return repository.missingHeaders, repository.missingHeadersErr
}
func (repository *MockDripFileRepoRepository) SetMarkHeaderCheckedErr(e error) {
repository.markHeaderCheckedErr = e
}
func (repository *MockDripFileRepoRepository) SetMissingHeadersErr(e error) {
repository.missingHeadersErr = e
}
@ -52,3 +65,6 @@ func (repository *MockDripFileRepoRepository) SetMissingHeaders(headers []core.H
func (repository *MockDripFileRepoRepository) SetCreateError(e error) {
repository.createErr = e
}
func (repository *MockDripFileRepoRepository) AssertMarkHeaderCheckedCalledWith(i int64) {
Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i))
}

View File

@ -22,12 +22,12 @@ import (
type MockDripFileVowConverter struct {
converterErr error
PassedLog types.Log
PassedLogs []types.Log
}
func (converter *MockDripFileVowConverter) ToModel(ethLog types.Log) (vow.DripFileVowModel, error) {
converter.PassedLog = ethLog
return test_data.DripFileVowModel, converter.converterErr
func (converter *MockDripFileVowConverter) ToModels(ethLogs []types.Log) ([]vow.DripFileVowModel, error) {
converter.PassedLogs = ethLogs
return []vow.DripFileVowModel{test_data.DripFileVowModel}, converter.converterErr
}
func (converter *MockDripFileVowConverter) SetConverterError(e error) {

View File

@ -15,32 +15,45 @@
package vow
import (
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/vow"
)
type MockDripFileVowRepository struct {
createErr error
missingHeaders []core.Header
missingHeadersErr error
PassedStartingBlockNumber int64
PassedEndingBlockNumber int64
PassedHeaderID int64
PassedModel vow.DripFileVowModel
createErr error
markHeaderCheckedErr error
markHeaderCheckedPassedHeaderID int64
missingHeaders []core.Header
missingHeadersErr error
PassedStartingBlockNumber int64
PassedEndingBlockNumber int64
PassedHeaderID int64
PassedModels []vow.DripFileVowModel
}
func (repository *MockDripFileVowRepository) Create(headerID int64, model vow.DripFileVowModel) error {
func (repository *MockDripFileVowRepository) Create(headerID int64, models []vow.DripFileVowModel) error {
repository.PassedHeaderID = headerID
repository.PassedModel = model
repository.PassedModels = models
return repository.createErr
}
func (repository *MockDripFileVowRepository) MarkHeaderChecked(headerID int64) error {
repository.markHeaderCheckedPassedHeaderID = headerID
return repository.markHeaderCheckedErr
}
func (repository *MockDripFileVowRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
repository.PassedStartingBlockNumber = startingBlockNumber
repository.PassedEndingBlockNumber = endingBlockNumber
return repository.missingHeaders, repository.missingHeadersErr
}
func (repository *MockDripFileVowRepository) SetMarkHeaderCheckedErr(e error) {
repository.markHeaderCheckedErr = e
}
func (repository *MockDripFileVowRepository) SetMissingHeadersErr(e error) {
repository.missingHeadersErr = e
}
@ -52,3 +65,6 @@ func (repository *MockDripFileVowRepository) SetMissingHeaders(headers []core.He
func (repository *MockDripFileVowRepository) SetCreateError(e error) {
repository.createErr = e
}
func (repository *MockDripFileVowRepository) AssertMarkHeaderCheckedCalledWith(i int64) {
Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i))
}

View File

@ -22,30 +22,28 @@ import (
)
type MockFrobConverter struct {
PassedContractAddress string
PassedContractABI string
PassedLog types.Log
PassedEntity frob.FrobEntity
toEntityError error
toModelError error
PassedContractABI string
PassedLogs []types.Log
PassedEntities []frob.FrobEntity
toEntityError error
toModelError error
}
func (converter *MockFrobConverter) SetToEntityError(err error) {
func (converter *MockFrobConverter) SetToEntitiesError(err error) {
converter.toEntityError = err
}
func (converter *MockFrobConverter) SetToModelError(err error) {
func (converter *MockFrobConverter) SetToModelsError(err error) {
converter.toModelError = err
}
func (converter *MockFrobConverter) ToEntity(contractAddress string, contractAbi string, ethLog types.Log) (frob.FrobEntity, error) {
converter.PassedContractAddress = contractAddress
func (converter *MockFrobConverter) ToEntities(contractAbi string, ethLogs []types.Log) ([]frob.FrobEntity, error) {
converter.PassedContractABI = contractAbi
converter.PassedLog = ethLog
return test_data.FrobEntity, converter.toEntityError
converter.PassedLogs = ethLogs
return []frob.FrobEntity{test_data.FrobEntity}, converter.toEntityError
}
func (converter *MockFrobConverter) ToModel(frobEntity frob.FrobEntity) (frob.FrobModel, error) {
converter.PassedEntity = frobEntity
return test_data.FrobModel, converter.toModelError
func (converter *MockFrobConverter) ToModels(frobEntities []frob.FrobEntity) ([]frob.FrobModel, error) {
converter.PassedEntities = frobEntities
return []frob.FrobModel{test_data.FrobModel}, converter.toModelError
}

View File

@ -15,24 +15,32 @@
package frob
import (
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/transformers/frob"
)
type MockFrobRepository struct {
createError error
PassedEndingBlockNumber int64
PassedFrobModel frob.FrobModel
PassedHeaderID int64
PassedStartingBlockNumber int64
missingHeaders []core.Header
missingHeadersErr error
createError error
PassedEndingBlockNumber int64
PassedModels []frob.FrobModel
PassedHeaderID int64
PassedStartingBlockNumber int64
markHeaderCheckedErr error
markHeaderCheckedPassedHeaderID int64
missingHeaders []core.Header
missingHeadersErr error
}
func (repository *MockFrobRepository) SetCreateError(err error) {
repository.createError = err
}
func (repository *MockFrobRepository) SetMarkHeaderCheckedErr(e error) {
repository.markHeaderCheckedErr = e
}
func (repository *MockFrobRepository) SetMissingHeadersErr(err error) {
repository.missingHeadersErr = err
}
@ -41,14 +49,22 @@ func (repository *MockFrobRepository) SetMissingHeaders(headers []core.Header) {
repository.missingHeaders = headers
}
func (repository *MockFrobRepository) Create(headerID int64, model frob.FrobModel) error {
func (repository *MockFrobRepository) Create(headerID int64, models []frob.FrobModel) error {
repository.PassedHeaderID = headerID
repository.PassedFrobModel = model
repository.PassedModels = models
return repository.createError
}
func (repository *MockFrobRepository) MarkHeaderChecked(headerID int64) error {
repository.markHeaderCheckedPassedHeaderID = headerID
return repository.markHeaderCheckedErr
}
func (repository *MockFrobRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
repository.PassedStartingBlockNumber = startingBlockNumber
repository.PassedEndingBlockNumber = endingBlockNumber
return repository.missingHeaders, repository.missingHeadersErr
}
func (repository *MockFrobRepository) AssertMarkHeaderCheckedCalledWith(i int64) {
Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i))
}

View File

@ -23,12 +23,12 @@ import (
type MockPitFileDebtCeilingConverter struct {
converterErr error
PassedLog types.Log
PassedLogs []types.Log
}
func (converter *MockPitFileDebtCeilingConverter) ToModel(ethLog types.Log) (debt_ceiling.PitFileDebtCeilingModel, error) {
converter.PassedLog = ethLog
return test_data.PitFileDebtCeilingModel, converter.converterErr
func (converter *MockPitFileDebtCeilingConverter) ToModels(ethLogs []types.Log) ([]debt_ceiling.PitFileDebtCeilingModel, error) {
converter.PassedLogs = ethLogs
return []debt_ceiling.PitFileDebtCeilingModel{test_data.PitFileDebtCeilingModel}, converter.converterErr
}
func (converter *MockPitFileDebtCeilingConverter) SetConverterError(e error) {

View File

@ -15,23 +15,32 @@
package debt_ceiling
import (
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/debt_ceiling"
)
type MockPitFileDebtCeilingRepository struct {
createErr error
missingHeaders []core.Header
missingHeadersErr error
PassedStartingBlockNumber int64
PassedEndingBlockNumber int64
PassedHeaderID int64
PassedModel debt_ceiling.PitFileDebtCeilingModel
createErr error
markHeaderCheckedErr error
markHeaderCheckedPassedHeaderID int64
missingHeaders []core.Header
missingHeadersErr error
PassedStartingBlockNumber int64
PassedEndingBlockNumber int64
PassedHeaderID int64
PassedModels []debt_ceiling.PitFileDebtCeilingModel
}
func (repository *MockPitFileDebtCeilingRepository) Create(headerID int64, model debt_ceiling.PitFileDebtCeilingModel) error {
func (repository *MockPitFileDebtCeilingRepository) MarkHeaderChecked(headerID int64) error {
repository.markHeaderCheckedPassedHeaderID = headerID
return repository.markHeaderCheckedErr
}
func (repository *MockPitFileDebtCeilingRepository) Create(headerID int64, models []debt_ceiling.PitFileDebtCeilingModel) error {
repository.PassedHeaderID = headerID
repository.PassedModel = model
repository.PassedModels = models
return repository.createErr
}
@ -41,6 +50,10 @@ func (repository *MockPitFileDebtCeilingRepository) MissingHeaders(startingBlock
return repository.missingHeaders, repository.missingHeadersErr
}
func (repository *MockPitFileDebtCeilingRepository) SetMarkHeaderCheckedErr(e error) {
repository.markHeaderCheckedErr = e
}
func (repository *MockPitFileDebtCeilingRepository) SetMissingHeadersErr(e error) {
repository.missingHeadersErr = e
}
@ -52,3 +65,7 @@ func (repository *MockPitFileDebtCeilingRepository) SetMissingHeaders(headers []
func (repository *MockPitFileDebtCeilingRepository) SetCreateError(e error) {
repository.createErr = e
}
func (repository *MockPitFileDebtCeilingRepository) AssertMarkHeaderCheckedCalledWith(i int64) {
Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i))
}

View File

@ -22,7 +22,7 @@ import (
)
type MockPitFileIlkConverter struct {
PassedLog types.Log
PassedLogs []types.Log
converterError error
}
@ -30,7 +30,7 @@ func (converter *MockPitFileIlkConverter) SetConverterError(err error) {
converter.converterError = err
}
func (converter *MockPitFileIlkConverter) ToModel(ethLog types.Log) (ilk.PitFileIlkModel, error) {
converter.PassedLog = ethLog
return test_data.PitFileIlkModel, converter.converterError
func (converter *MockPitFileIlkConverter) ToModels(ethLogs []types.Log) ([]ilk.PitFileIlkModel, error) {
converter.PassedLogs = ethLogs
return []ilk.PitFileIlkModel{test_data.PitFileIlkModel}, converter.converterError
}

View File

@ -15,24 +15,32 @@
package ilk
import (
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/ilk"
)
type MockPitFileIlkRepository struct {
createError error
PassedEndingBlockNumber int64
PassedModel ilk.PitFileIlkModel
PassedHeaderID int64
PassedStartingBlockNumber int64
missingHeaders []core.Header
missingHeadersErr error
createError error
PassedEndingBlockNumber int64
PassedModels []ilk.PitFileIlkModel
PassedHeaderID int64
PassedStartingBlockNumber int64
markHeaderCheckedErr error
markHeaderCheckedPassedHeaderID int64
missingHeaders []core.Header
missingHeadersErr error
}
func (repository *MockPitFileIlkRepository) SetCreateError(err error) {
repository.createError = err
}
func (repository *MockPitFileIlkRepository) SetMarkHeaderCheckedErr(e error) {
repository.markHeaderCheckedErr = e
}
func (repository *MockPitFileIlkRepository) SetMissingHeadersErr(err error) {
repository.missingHeadersErr = err
}
@ -41,14 +49,23 @@ func (repository *MockPitFileIlkRepository) SetMissingHeaders(headers []core.Hea
repository.missingHeaders = headers
}
func (repository *MockPitFileIlkRepository) Create(headerID int64, model ilk.PitFileIlkModel) error {
func (repository *MockPitFileIlkRepository) Create(headerID int64, models []ilk.PitFileIlkModel) error {
repository.PassedHeaderID = headerID
repository.PassedModel = model
repository.PassedModels = models
return repository.createError
}
func (repository *MockPitFileIlkRepository) MarkHeaderChecked(headerID int64) error {
repository.markHeaderCheckedPassedHeaderID = headerID
return repository.markHeaderCheckedErr
}
func (repository *MockPitFileIlkRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
repository.PassedStartingBlockNumber = startingBlockNumber
repository.PassedEndingBlockNumber = endingBlockNumber
return repository.missingHeaders, repository.missingHeadersErr
}
func (repository *MockPitFileIlkRepository) AssertMarkHeaderCheckedCalledWith(i int64) {
Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i))
}

View File

@ -23,13 +23,14 @@ import (
type MockPitFileStabilityFeeConverter struct {
converterErr error
PassedLog types.Log
PassedLogs []types.Log
}
func (converter *MockPitFileStabilityFeeConverter) ToModel(ethLog types.Log) (stability_fee.PitFileStabilityFeeModel, error) {
converter.PassedLog = ethLog
return test_data.PitFileStabilityFeeModel, converter.converterErr
func (converter *MockPitFileStabilityFeeConverter) ToModels(ethLogs []types.Log) ([]stability_fee.PitFileStabilityFeeModel, error) {
converter.PassedLogs = ethLogs
return []stability_fee.PitFileStabilityFeeModel{test_data.PitFileStabilityFeeModel}, converter.converterErr
}
func (converter *MockPitFileStabilityFeeConverter) SetConverterError(e error) {
converter.converterErr = e
}

View File

@ -15,38 +15,57 @@
package stability_fee
import (
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/stability_fee"
)
type MockPitFileStabilityFeeRepository struct {
createErr error
missingHeaders []core.Header
missingHeadersErr error
PassedStartingBlockNumber int64
PassedEndingBlockNumber int64
PassedHeaderID int64
PassedModel stability_fee.PitFileStabilityFeeModel
createErr error
markHeaderCheckedErr error
markHeaderCheckedPassedHeaderID int64
missingHeaders []core.Header
missingHeadersErr error
PassedStartingBlockNumber int64
PassedEndingBlockNumber int64
PassedHeaderID int64
PassedModels []stability_fee.PitFileStabilityFeeModel
}
func (repository *MockPitFileStabilityFeeRepository) Create(headerID int64, model stability_fee.PitFileStabilityFeeModel) error {
repository.PassedModel = model
func (repository *MockPitFileStabilityFeeRepository) Create(headerID int64, models []stability_fee.PitFileStabilityFeeModel) error {
repository.PassedModels = models
repository.PassedHeaderID = headerID
return repository.createErr
}
func (repository *MockPitFileStabilityFeeRepository) MarkHeaderChecked(headerID int64) error {
repository.markHeaderCheckedPassedHeaderID = headerID
return repository.markHeaderCheckedErr
}
func (repository *MockPitFileStabilityFeeRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
repository.PassedStartingBlockNumber = startingBlockNumber
repository.PassedEndingBlockNumber = endingBlockNumber
return repository.missingHeaders, repository.missingHeadersErr
}
func (repository *MockPitFileStabilityFeeRepository) SetMarkHeaderCheckedErr(e error) {
repository.markHeaderCheckedErr = e
}
func (repository *MockPitFileStabilityFeeRepository) SetMissingHeadersErr(e error) {
repository.missingHeadersErr = e
}
func (repository *MockPitFileStabilityFeeRepository) SetMissingHeaders(headers []core.Header) {
repository.missingHeaders = headers
}
func (repository *MockPitFileStabilityFeeRepository) SetCreateError(e error) {
repository.createErr = e
}
func (repository *MockPitFileStabilityFeeRepository) AssertMarkHeaderCheckedCalledWith(i int64) {
Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i))
}

View File

@ -23,12 +23,12 @@ import (
type MockVatInitConverter struct {
converterErr error
PassedLog types.Log
PassedLogs []types.Log
}
func (converter *MockVatInitConverter) ToModel(ethLog types.Log) (vat_init.VatInitModel, error) {
converter.PassedLog = ethLog
return test_data.VatInitModel, converter.converterErr
func (converter *MockVatInitConverter) ToModels(ethLogs []types.Log) ([]vat_init.VatInitModel, error) {
converter.PassedLogs = ethLogs
return []vat_init.VatInitModel{test_data.VatInitModel}, converter.converterErr
}
func (converter *MockVatInitConverter) SetConverterError(e error) {

View File

@ -15,32 +15,45 @@
package vat_init
import (
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/transformers/vat_init"
)
type MockVatInitRepository struct {
createErr error
missingHeaders []core.Header
missingHeadersErr error
PassedStartingBlockNumber int64
PassedEndingBlockNumber int64
PassedHeaderID int64
PassedModel vat_init.VatInitModel
createErr error
markHeaderCheckedErr error
markHeaderCheckedPassedHeaderID int64
missingHeaders []core.Header
missingHeadersErr error
PassedStartingBlockNumber int64
PassedEndingBlockNumber int64
PassedHeaderID int64
PassedModels []vat_init.VatInitModel
}
func (repository *MockVatInitRepository) Create(headerID int64, model vat_init.VatInitModel) error {
func (repository *MockVatInitRepository) Create(headerID int64, models []vat_init.VatInitModel) error {
repository.PassedHeaderID = headerID
repository.PassedModel = model
repository.PassedModels = models
return repository.createErr
}
func (repository *MockVatInitRepository) MarkHeaderChecked(headerID int64) error {
repository.markHeaderCheckedPassedHeaderID = headerID
return repository.markHeaderCheckedErr
}
func (repository *MockVatInitRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
repository.PassedStartingBlockNumber = startingBlockNumber
repository.PassedEndingBlockNumber = endingBlockNumber
return repository.missingHeaders, repository.missingHeadersErr
}
func (repository *MockVatInitRepository) SetMarkHeaderCheckedErr(e error) {
repository.markHeaderCheckedErr = e
}
func (repository *MockVatInitRepository) SetMissingHeadersErr(e error) {
repository.missingHeadersErr = e
}
@ -52,3 +65,7 @@ func (repository *MockVatInitRepository) SetMissingHeaders(headers []core.Header
func (repository *MockVatInitRepository) SetCreateError(e error) {
repository.createErr = e
}
func (repository *MockVatInitRepository) AssertMarkHeaderCheckedCalledWith(i int64) {
Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i))
}

View File

@ -23,23 +23,31 @@ import (
)
type Converter interface {
ToModel(ethLog types.Log) (VatInitModel, error)
ToModels(ethLogs []types.Log) ([]VatInitModel, error)
}
type VatInitConverter struct{}
func (VatInitConverter) ToModel(ethLog types.Log) (VatInitModel, error) {
err := verifyLog(ethLog)
if err != nil {
return VatInitModel{}, err
func (VatInitConverter) ToModels(ethLogs []types.Log) ([]VatInitModel, error) {
var models []VatInitModel
for _, ethLog := range ethLogs {
err := verifyLog(ethLog)
if err != nil {
return nil, err
}
ilk := string(bytes.Trim(ethLog.Topics[1].Bytes(), "\x00"))
raw, err := json.Marshal(ethLog)
if err != nil {
return nil, err
}
model := VatInitModel{
Ilk: ilk,
TransactionIndex: ethLog.TxIndex,
Raw: raw,
}
models = append(models, model)
}
ilk := string(bytes.Trim(ethLog.Topics[1].Bytes(), "\x00"))
raw, err := json.Marshal(ethLog)
return VatInitModel{
Ilk: ilk,
TransactionIndex: ethLog.TxIndex,
Raw: raw,
}, err
return models, nil
}
func verifyLog(log types.Log) error {

View File

@ -28,7 +28,7 @@ var _ = Describe("Vat init converter", func() {
converter := vat_init.VatInitConverter{}
badLog := types.Log{}
_, err := converter.ToModel(badLog)
_, err := converter.ToModels([]types.Log{badLog})
Expect(err).To(HaveOccurred())
})
@ -36,9 +36,10 @@ var _ = Describe("Vat init converter", func() {
It("converts a log to an model", func() {
converter := vat_init.VatInitConverter{}
model, err := converter.ToModel(test_data.EthVatInitLog)
models, err := converter.ToModels([]types.Log{test_data.EthVatInitLog})
Expect(err).NotTo(HaveOccurred())
Expect(model).To(Equal(test_data.VatInitModel))
Expect(len(models)).To(Equal(1))
Expect(models[0]).To(Equal(test_data.VatInitModel))
})
})

View File

@ -20,7 +20,8 @@ import (
)
type Repository interface {
Create(headerID int64, model VatInitModel) error
Create(headerID int64, models []VatInitModel) error
MarkHeaderChecked(headerID int64) error
MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
}
@ -34,12 +35,31 @@ func NewVatInitRepository(db *postgres.DB) VatInitRepository {
}
}
func (repository VatInitRepository) Create(headerID int64, model VatInitModel) error {
_, err := repository.db.Exec(
`INSERT into maker.vat_init (header_id, ilk, tx_idx, raw_log)
func (repository VatInitRepository) Create(headerID int64, models []VatInitModel) error {
tx, err := repository.db.Begin()
if err != nil {
return err
}
for _, model := range models {
_, err = tx.Exec(
`INSERT into maker.vat_init (header_id, ilk, tx_idx, raw_log)
VALUES($1, $2, $3, $4)`,
headerID, model.Ilk, model.TransactionIndex, model.Raw,
)
headerID, model.Ilk, model.TransactionIndex, model.Raw,
)
if err != nil {
tx.Rollback()
return err
}
}
return tx.Commit()
}
func (repository VatInitRepository) MarkHeaderChecked(headerID int64) error {
_, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, vat_init_checked)
VALUES ($1, $2)
ON CONFLICT (header_id) DO
UPDATE SET vat_init_checked = $2`, headerID, true)
return err
}
@ -48,8 +68,8 @@ func (repository VatInitRepository) MissingHeaders(startingBlockNumber, endingBl
err := repository.db.Select(
&result,
`SELECT headers.id, headers.block_number FROM headers
LEFT JOIN maker.vat_init on headers.id = header_id
WHERE header_id ISNULL
LEFT JOIN checked_headers on headers.id = header_id
WHERE (header_id ISNULL OR vat_init_checked IS FALSE)
AND headers.block_number >= $1
AND headers.block_number <= $2
AND headers.eth_node_fingerprint = $3`,
@ -57,6 +77,5 @@ func (repository VatInitRepository) MissingHeaders(startingBlockNumber, endingBl
endingBlockNumber,
repository.db.Node.ID,
)
return result, err
}

View File

@ -21,13 +21,14 @@ import (
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
"github.com/vulcanize/vulcanizedb/pkg/transformers/vat_init"
"github.com/vulcanize/vulcanizedb/test_config"
)
var _ = Describe("", func() {
var _ = Describe("Vat init repository", func() {
Describe("Create", func() {
It("adds a vat event", func() {
db := test_config.NewTestDB(core.Node{})
@ -37,7 +38,7 @@ var _ = Describe("", func() {
Expect(err).NotTo(HaveOccurred())
vatInitRepository := vat_init.NewVatInitRepository(db)
err = vatInitRepository.Create(headerID, test_data.VatInitModel)
err = vatInitRepository.Create(headerID, []vat_init.VatInitModel{test_data.VatInitModel})
Expect(err).NotTo(HaveOccurred())
var dbVatInit vat_init.VatInitModel
@ -55,10 +56,10 @@ var _ = Describe("", func() {
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
vatInitRepository := vat_init.NewVatInitRepository(db)
err = vatInitRepository.Create(headerID, test_data.VatInitModel)
err = vatInitRepository.Create(headerID, []vat_init.VatInitModel{test_data.VatInitModel})
Expect(err).NotTo(HaveOccurred())
err = vatInitRepository.Create(headerID, test_data.VatInitModel)
err = vatInitRepository.Create(headerID, []vat_init.VatInitModel{test_data.VatInitModel})
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
@ -71,7 +72,7 @@ var _ = Describe("", func() {
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
vatInitRepository := vat_init.NewVatInitRepository(db)
err = vatInitRepository.Create(headerID, test_data.VatInitModel)
err = vatInitRepository.Create(headerID, []vat_init.VatInitModel{test_data.VatInitModel})
Expect(err).NotTo(HaveOccurred())
_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
@ -84,8 +85,48 @@ var _ = Describe("", func() {
})
})
Describe("MarkHeaderChecked", func() {
var (
db *postgres.DB
vatInitRepository vat_init.Repository
err error
headerID int64
)
BeforeEach(func() {
db = test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
vatInitRepository = vat_init.NewVatInitRepository(db)
})
It("creates a row for a new headerID", func() {
err = vatInitRepository.MarkHeaderChecked(headerID)
Expect(err).NotTo(HaveOccurred())
var headerChecked bool
err = db.Get(&headerChecked, `SELECT vat_init_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue())
})
It("updates row when headerID already exists", func() {
_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID)
err = vatInitRepository.MarkHeaderChecked(headerID)
Expect(err).NotTo(HaveOccurred())
var headerChecked bool
err = db.Get(&headerChecked, `SELECT vat_init_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue())
})
})
Describe("MissingHeaders", func() {
It("returns headers with no associated vat event", func() {
It("returns headers that haven't been checked", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
@ -100,7 +141,7 @@ var _ = Describe("", func() {
Expect(err).NotTo(HaveOccurred())
}
vatInitRepository := vat_init.NewVatInitRepository(db)
err := vatInitRepository.Create(headerIDs[1], test_data.VatInitModel)
err := vatInitRepository.MarkHeaderChecked(headerIDs[1])
Expect(err).NotTo(HaveOccurred())
headers, err := vatInitRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
@ -111,6 +152,33 @@ var _ = Describe("", func() {
Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
})
It("only treats headers as checked if drip drip logs have been checked", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
startingBlockNumber := int64(1)
vatInitBlockNumber := int64(2)
endingBlockNumber := int64(3)
blockNumbers := []int64{startingBlockNumber, vatInitBlockNumber, endingBlockNumber, endingBlockNumber + 1}
var headerIDs []int64
for _, n := range blockNumbers {
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
headerIDs = append(headerIDs, headerID)
Expect(err).NotTo(HaveOccurred())
}
VatInitRepository := vat_init.NewVatInitRepository(db)
_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
Expect(err).NotTo(HaveOccurred())
headers, err := VatInitRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
Expect(err).NotTo(HaveOccurred())
Expect(len(headers)).To(Equal(3))
Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatInitBlockNumber)))
Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatInitBlockNumber)))
Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatInitBlockNumber)))
})
It("only returns headers associated with the current node", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
@ -128,7 +196,7 @@ var _ = Describe("", func() {
}
vatInitRepository := vat_init.NewVatInitRepository(db)
vatInitRepositoryTwo := vat_init.NewVatInitRepository(dbTwo)
err := vatInitRepository.Create(headerIDs[0], test_data.VatInitModel)
err := vatInitRepository.MarkHeaderChecked(headerIDs[0])
Expect(err).NotTo(HaveOccurred())
nodeOneMissingHeaders, err := vatInitRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])

View File

@ -59,16 +59,20 @@ func (transformer VatInitTransformer) Execute() error {
if err != nil {
return err
}
for _, log := range matchingLogs {
model, err := transformer.Converter.ToModel(log)
if err != nil {
return err
}
err = transformer.Repository.Create(header.Id, model)
if len(matchingLogs) < 1 {
err = transformer.Repository.MarkHeaderChecked(header.Id)
if err != nil {
return err
}
}
models, err := transformer.Converter.ToModels(matchingLogs)
if err != nil {
return err
}
err = transformer.Repository.Create(header.Id, models)
if err != nil {
return err
}
}
return nil
}

View File

@ -96,6 +96,42 @@ var _ = Describe("Vat init transformer", func() {
Expect(err).To(MatchError(fakes.FakeError))
})
It("marks header checked if no logs returned", func() {
mockConverter := &vat_init_mocks.MockVatInitConverter{}
mockRepository := &vat_init_mocks.MockVatInitRepository{}
headerID := int64(123)
mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}})
mockFetcher := &mocks.MockLogFetcher{}
transformer := vat_init.VatInitTransformer{
Converter: mockConverter,
Fetcher: mockFetcher,
Repository: mockRepository,
}
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
mockRepository.AssertMarkHeaderCheckedCalledWith(headerID)
})
It("returns error if marking header checked returns err", func() {
mockConverter := &vat_init_mocks.MockVatInitConverter{}
mockRepository := &vat_init_mocks.MockVatInitRepository{}
mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}})
mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError)
mockFetcher := &mocks.MockLogFetcher{}
transformer := vat_init.VatInitTransformer{
Converter: mockConverter,
Fetcher: mockFetcher,
Repository: mockRepository,
}
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("converts matching logs", func() {
converter := &vat_init_mocks.MockVatInitConverter{}
fetcher := &mocks.MockLogFetcher{}
@ -111,7 +147,7 @@ var _ = Describe("Vat init transformer", func() {
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(converter.PassedLog).To(Equal(test_data.EthVatInitLog))
Expect(converter.PassedLogs).To(Equal([]types.Log{test_data.EthVatInitLog}))
})
It("returns error if converter returns error", func() {
@ -150,7 +186,7 @@ var _ = Describe("Vat init transformer", func() {
Expect(err).NotTo(HaveOccurred())
Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id))
Expect(repository.PassedModel).To(Equal(test_data.VatInitModel))
Expect(repository.PassedModels).To(Equal([]vat_init.VatInitModel{test_data.VatInitModel}))
})
It("returns error if repository returns error for create", func() {