cerc-io/ipld-eth-beacon-indexer
19
0
Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases 9 Wiki Activity

76: Add indexing of ExecutionPayloads (and other Merge-related updates). #73

Merged
telackey merged 30 commits from telackey/the_merge into main 2022-09-29 01:39:56 +00:00
Conversation 0 Commits 30 Files Changed 38 +162 -55
5 changed files with 162 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 6f2de13426 - Show all commits

105
pkg/beaconclient/capturehead_test.go
View File

@ -192,10 +192,14 @@ var (
},
"2375703": {
HeadMessage: beaconclient.Head{
Slot: "2375703",
Block: "0x4392372c5f6e39499e31bf924388b5815639103149f0f54f8a453773b1802301",
State: "0xb6215b560273af63ec7e011572b60ec1ca0b0232f8ff44fcd4ed55c7526e964e",
CurrentDutyDependentRoot: "", PreviousDutyDependentRoot: "", EpochTransition: false, ExecutionOptimistic: false},
Slot: "2375703",
Block: "0x4392372c5f6e39499e31bf924388b5815639103149f0f54f8a453773b1802301",
State: "0xb6215b560273af63ec7e011572b60ec1ca0b0232f8ff44fcd4ed55c7526e964e",
CurrentDutyDependentRoot: "",
PreviousDutyDependentRoot: "",
EpochTransition: false,
ExecutionOptimistic: false,
},
TestNotes: "An easy to process Altair Block",
SignedBeaconBlock: filepath.Join("ssz-data", "2375703", "signed-beacon-block.ssz"),
BeaconState: filepath.Join("ssz-data", "2375703", "beacon-state.ssz"),
@ -206,15 +210,73 @@ var (
},
"3797056": {
HeadMessage: beaconclient.Head{
Slot: "3797056",
Block: "",
State: "",
CurrentDutyDependentRoot: "", PreviousDutyDependentRoot: "", EpochTransition: false, ExecutionOptimistic: false},
Slot: "3797056",
Block: "",
State: "",
CurrentDutyDependentRoot: "",
PreviousDutyDependentRoot: "",
EpochTransition: false,
ExecutionOptimistic: false,
},
TestNotes: "An easy to process Altair Block",
// The file below should not exist, this will trigger an error message and 404 response from the mock.
SignedBeaconBlock: filepath.Join("ssz-data", "3797056", "should-not-exist.txt"),
BeaconState: filepath.Join("ssz-data", "3797056", "beacon-state.ssz"),
},
"4636671": {
HeadMessage: beaconclient.Head{
Slot: "4636671",
Block: "0xe7d4f3b7924c30ae047fceabb853b8afdae32b85e0a87ab6c4c37421b353a1da",
State: "0x66146a0bc8656a63aaf5dd357f327cac58c83fc90582ced82bebcc6e5f11855b",
CurrentDutyDependentRoot: "",
PreviousDutyDependentRoot: "",
EpochTransition: false,
ExecutionOptimistic: false,
},
TestNotes: "The last Altair block",
SignedBeaconBlock: filepath.Join("ssz-data", "4636671", "signed-beacon-block.ssz"),
BeaconState: filepath.Join("ssz-data", "4636671", "beacon-state.ssz"),
CorrectEth1DataBlockHash: "0xa5b11e0cfb9ffd53e298f0d24fe07bc7a19ada6e52fa3f09397e1b34c07b4ec6",
CorrectParentRoot: "0x47fc3b7a28512a2570438c02bd0b96ebcac8bbcd97eed6d50f15454f37ac51b8",
CorrectSignedBeaconBlockMhKey: "",
CorrectBeaconStateMhKey: "",
},
"4636672": {
HeadMessage: beaconclient.Head{
Slot: "4636672",
Block: "0x9429ce339da8944dd2e1565be8cac5bf634cae2120b6937c081e39148a7f4b1a",
State: "0x0067a5d28b38e6e2f59a73046fabbf16a782b978c2c89621a679e7f682b05bd4",
CurrentDutyDependentRoot: "",
PreviousDutyDependentRoot: "",
EpochTransition: true,
ExecutionOptimistic: false,
},
TestNotes: "The first Bellatrix block (empty ExecutionPayload)",
SignedBeaconBlock: filepath.Join("ssz-data", "4636672", "signed-beacon-block.ssz"),
BeaconState: filepath.Join("ssz-data", "4636672", "beacon-state.ssz"),
CorrectEth1DataBlockHash: "0x3b7d392e46db19704d677cadb3310c3776d8c0b8cb2af1c324bb4a394b7f8164",
CorrectParentRoot: "0xe7d4f3b7924c30ae047fceabb853b8afdae32b85e0a87ab6c4c37421b353a1da",
CorrectSignedBeaconBlockMhKey: "/blocks/QLVAEQRQPA4TIMRZMNSTGMZZMRQTQOJUGRSGIMTFGE2TMNLCMU4GGYLDGVRGMNRTGRRWCZJSGEZDAYRWHEZTOYZQHAYWKMZZGE2DQYJXMY2GEMLB",
CorrectBeaconStateMhKey: "",
},
"4700013": {
HeadMessage: beaconclient.Head{
Slot: "4700013",
Block: "0x810a00400a80cdffc11ffdcf17ac404ac4dba215b95221955a9dfddf163d0b0d",
State: "0x171ef131e0638eddfe1ef73e7b483e344b1cf128b092f2c39e946eb7775b3a2f",
CurrentDutyDependentRoot: "",
PreviousDutyDependentRoot: "",
EpochTransition: true,
ExecutionOptimistic: false,
},
TestNotes: "The first Bellatrix block post-Merge (with ExecutionPayload)",
SignedBeaconBlock: filepath.Join("ssz-data", "4700013", "signed-beacon-block.ssz"),
BeaconState: filepath.Join("ssz-data", "4700013", "beacon-state.ssz"),
CorrectEth1DataBlockHash: "0xb8736ada384707e156f2e0e69d8311ceda11f96806921644a378fd55899894ca",
CorrectParentRoot: "0x60e751f7d2cf0ae24b195bda37e9add56a7d8c4b75469c018c0f912518c3bae8",
CorrectSignedBeaconBlockMhKey: "/blocks/QLVAEQRQPA4DCMDBGAYDIMBQME4DAY3EMZTGGMJRMZTGIY3GGE3WCYZUGA2GCYZUMRRGCMRRGVRDSNJSGIYTSNJVME4WIZTEMRTDCNRTMQYGEMDE",
CorrectBeaconStateMhKey: "",
},
}
TestConfig = Config{
protocol: protocol,
@ -303,6 +365,20 @@ var _ = Describe("Capturehead", Label("head"), func() {
})
})
Context("Correctly formatted Bellatrix Test Blocks", Label("unit", "bellatrix"), func() {
It("Should turn it into a struct successfully (pre-Merge).", func() {
bc := setUpTest(BeaconNodeTester.TestConfig, "4636672")
BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
defer httpmock.DeactivateAndReset()
BeaconNodeTester.testProcessBlock(bc, BeaconNodeTester.TestEvents["4636672"].HeadMessage, 144896, maxRetry, 1, 0, 0)
})
It("Should turn it into a struct successfully (post-Merge).", func() {
bc := setUpTest(BeaconNodeTester.TestConfig, "4700013")
BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
defer httpmock.DeactivateAndReset()
BeaconNodeTester.testProcessBlock(bc, BeaconNodeTester.TestEvents["4700013"].HeadMessage, 146875, maxRetry, 1, 0, 0)
})
})
Context("Correctly formatted Phase0 Test Blocks", func() {
It("Should turn it into a struct successfully.", func() {
bc := setUpTest(BeaconNodeTester.TestConfig, "99")
@ -315,7 +391,6 @@ var _ = Describe("Capturehead", Label("head"), func() {
defer httpmock.DeactivateAndReset()
BeaconNodeTester.testProcessBlock(bc, BeaconNodeTester.TestEvents["100-dummy-2"].HeadMessage, 3, maxRetry, 1, 0, 0)
})
})
Context("Two consecutive correct blocks", func() {
It("Should handle both blocks correctly, without any reorgs or known_gaps", func() {
@ -568,11 +643,15 @@ func queryDbSlotAndBlock(db sql.Database, querySlot string, queryBlockRoot strin
// A helper function to query the eth_beacon.signed_block table based on the slot and block_root.
func queryDbSignedBeaconBlock(db sql.Database, querySlot string, queryBlockRoot string) (int, string, string, string, string) {
sqlStatement := `SELECT slot, block_root, parent_block_root, eth1_data_block_hash, mh_key FROM eth_beacon.signed_block WHERE slot=$1 AND block_root=$2;`
var slot int
var blockRoot, parentBlockRoot, eth1DataBlockHash, mhKey string
sqlStatement := `SELECT slot, block_root, parent_block_root, eth1_data_block_hash, mh_key,
payload_block_number, payload_timestamp, payload_block_hash,
payload_parent_hash, payload_state_root payload_receipts_root FROM eth_beacon.signed_block WHERE slot=$1 AND block_root=$2;`
var slot, payloadBlockNumber, payloadTimestamp int
var blockRoot, parentBlockRoot, eth1DataBlockHash, mhKey, payloadBlockHash, payloadParentHash, payloadStateRoot, payloadReceiptsRoot string
row := db.QueryRow(context.Background(), sqlStatement, querySlot, queryBlockRoot)
err := row.Scan(&slot, &blockRoot, &parentBlockRoot, &eth1DataBlockHash, &mhKey)
err := row.Scan(&slot, &blockRoot, &parentBlockRoot, &eth1DataBlockHash, &mhKey,
&payloadBlockNumber, &payloadTimestamp, &payloadBlockHash, &payloadParentHash,
&payloadStateRoot, &payloadReceiptsRoot)
Expect(err).ToNot(HaveOccurred())
return slot, blockRoot, parentBlockRoot, eth1DataBlockHash, mhKey
}

10
pkg/beaconclient/consensus.go
View File

@ -17,6 +17,7 @@ type Eth1Data common.Eth1Data
type Root common.Root
type Signature common.BLSSignature
type Slot common.Slot
type ExecutionPayloadHeader common.ExecutionPayloadHeader
type BeaconBlock struct {
spec *common.Spec
@ -266,6 +267,15 @@ func (b *BeaconBlockBody) Eth1Data() Eth1Data {
return Eth1Data{}
}
func (b *BeaconBlockBody) ExecutionPayloadHeader() *ExecutionPayloadHeader {
if b.IsBellatrix() {
payloadHeader := b.bellatrix.ExecutionPayload.Header(chooseSpec(b.spec))
return (*ExecutionPayloadHeader)(payloadHeader)
}
return nil
}
func (b *BeaconBlock) HashTreeRoot() Root {
spec := chooseSpec(b.spec)
hashFn := tree.GetHashFn()

51
pkg/beaconclient/databasewrite.go
View File

@ -39,8 +39,9 @@ VALUES ($1, $2, $3, $4, $5) ON CONFLICT (slot, block_root) DO NOTHING`
UpsertSignedBeaconBlockWithPayloadStmt string = `
INSERT INTO eth_beacon.signed_block (slot, block_root, parent_block_root, eth1_data_block_hash, mh_key,
payload_block_number, payload_timestamp, payload_block_hash,
payload_parent_hash, payload_state_root, payload_receipts_root)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) ON CONFLICT (slot, block_root) DO NOTHING`
payload_parent_hash, payload_state_root, payload_receipts_root,
payload_transactions_root)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) ON CONFLICT (slot, block_root) DO NOTHING`
// Statement to upsert to the eth_beacon.state table.
UpsertBeaconState string = `
INSERT INTO eth_beacon.state (slot, state_root, mh_key)
@ -100,7 +101,7 @@ type DatabaseWriter struct {
}
func CreateDatabaseWrite(db sql.Database, slot int, stateRoot string, blockRoot string, parentBlockRoot string,
eth1DataBlockHash string, payloadSummary *ExecutionPayloadSummary, status string, rawSignedBeaconBlock *[]byte, rawBeaconState *[]byte, metrics *BeaconClientMetrics) (*DatabaseWriter, error) {
eth1DataBlockHash string, payloadHeader *ExecutionPayloadHeader, status string, rawSignedBeaconBlock *[]byte, rawBeaconState *[]byte, metrics *BeaconClientMetrics) (*DatabaseWriter, error) {
ctx := context.Background()
tx, err := db.Begin(ctx)
if err != nil {
@ -115,7 +116,7 @@ func CreateDatabaseWrite(db sql.Database, slot int, stateRoot string, blockRoot
Metrics: metrics,
}
dw.prepareSlotsModel(slot, stateRoot, blockRoot, status)
err = dw.prepareSignedBeaconBlockModel(slot, blockRoot, parentBlockRoot, eth1DataBlockHash, payloadSummary)
err = dw.prepareSignedBeaconBlockModel(slot, blockRoot, parentBlockRoot, eth1DataBlockHash, payloadHeader)
if err != nil {
return nil, err
}
@ -143,19 +144,32 @@ func (dw *DatabaseWriter) prepareSlotsModel(slot int, stateRoot string, blockRoo
// Create the model for the eth_beacon.signed_block table.
func (dw *DatabaseWriter) prepareSignedBeaconBlockModel(slot int, blockRoot string, parentBlockRoot string, eth1DataBlockHash string,
payloadSummary *ExecutionPayloadSummary) error {
payloadHeader *ExecutionPayloadHeader) error {
mhKey, err := MultihashKeyFromSSZRoot([]byte(dw.DbSlots.BlockRoot))
if err != nil {
return err
}
dw.DbSignedBeaconBlock = &DbSignedBeaconBlock{
Slot: strconv.Itoa(slot),
BlockRoot: blockRoot,
ParentBlock: parentBlockRoot,
Eth1DataBlockHash: eth1DataBlockHash,
MhKey: mhKey,
ExecutionPayload: payloadSummary,
Slot: strconv.Itoa(slot),
BlockRoot: blockRoot,
ParentBlock: parentBlockRoot,
Eth1DataBlockHash: eth1DataBlockHash,
MhKey: mhKey,
ExecutionPayloadHeader: nil,
}
if nil != payloadHeader {
dw.DbSignedBeaconBlock.ExecutionPayloadHeader = &DbExecutionPayloadHeader{
BlockNumber: uint64(payloadHeader.BlockNumber),
Timestamp: uint64(payloadHeader.Timestamp),
BlockHash: toHex(payloadHeader.BlockHash),
ParentHash: toHex(payloadHeader.ParentHash),
StateRoot: toHex(payloadHeader.StateRoot),
ReceiptsRoot: toHex(payloadHeader.ReceiptsRoot),
TransactionsRoot: toHex(payloadHeader.TransactionsRoot),
}
}
log.Debug("dw.DbSignedBeaconBlock: ", dw.DbSignedBeaconBlock)
return nil
}
@ -266,7 +280,7 @@ func (dw *DatabaseWriter) upsertPublicBlocks(key string, data *[]byte) error {
func (dw *DatabaseWriter) upsertSignedBeaconBlock() error {
block := dw.DbSignedBeaconBlock
var err error
if nil != block.ExecutionPayload {
if nil != block.ExecutionPayloadHeader {
_, err = dw.Tx.Exec(dw.Ctx,
UpsertSignedBeaconBlockWithPayloadStmt,
block.Slot,
@ -274,12 +288,13 @@ func (dw *DatabaseWriter) upsertSignedBeaconBlock() error {
block.ParentBlock,
block.Eth1DataBlockHash,
block.MhKey,
block.ExecutionPayload.PayloadBlockNumber,
block.ExecutionPayload.PayloadTimestamp,
block.ExecutionPayload.PayloadBlockHash,
block.ExecutionPayload.PayloadParentHash,
block.ExecutionPayload.PayloadStateRoot,
block.ExecutionPayload.PayloadReceiptsRoot,
block.ExecutionPayloadHeader.BlockNumber,
block.ExecutionPayloadHeader.Timestamp,
block.ExecutionPayloadHeader.BlockHash,
block.ExecutionPayloadHeader.ParentHash,
block.ExecutionPayloadHeader.StateRoot,
block.ExecutionPayloadHeader.ReceiptsRoot,
block.ExecutionPayloadHeader.TransactionsRoot,
)
} else {
_, err = dw.Tx.Exec(dw.Ctx,

28
pkg/beaconclient/models.go
View File

@ -60,23 +60,25 @@ type DbSlots struct {
Status string // The status, it can be proposed | forked | skipped.
}
type ExecutionPayloadSummary struct {
PayloadBlockNumber uint64
PayloadTimestamp uint64
PayloadBlockHash string
PayloadParentHash string
PayloadStateRoot string
PayloadReceiptsRoot string
// A struct to handle the details of an embedded Eth1-block (ie, the ExecutionPayload)
type DbExecutionPayloadHeader struct {
BlockNumber uint64
Timestamp uint64
BlockHash string
ParentHash string
StateRoot string
ReceiptsRoot string
TransactionsRoot string
}
// A struct to capture whats being written to eth-beacon.signed_block table.
type DbSignedBeaconBlock struct {
Slot string // The slot.
BlockRoot string // The block root
ParentBlock string // The parent block root.
Eth1DataBlockHash string // The eth1 block_hash
MhKey string // The ipld multihash key.
ExecutionPayload *ExecutionPayloadSummary
Slot string // The slot.
BlockRoot string // The block root
ParentBlock string // The parent block root.
Eth1DataBlockHash string // The eth1 block_hash
MhKey string // The ipld multihash key.
ExecutionPayloadHeader *DbExecutionPayloadHeader // The ExecutionPayloadHeader (after Bellatrix only).
}
// A struct to capture whats being written to eth-beacon.state table.

23
pkg/beaconclient/processslot.go
View File

@ -424,10 +424,10 @@ func (ps *ProcessSlot) createWriteObjects() (*DatabaseWriter, error) {
}
ps.PerformanceMetrics.ParseBeaconObjectForHash = time.Since(parseBeaconTime)
payloadSummary := ps.provideExecutionPayloadDetails()
payloadHeader := ps.provideExecutionPayloadDetails()
dw, err := CreateDatabaseWrite(ps.Db, ps.Slot, stateRoot, blockRoot, ps.ParentBlockRoot, eth1DataBlockHash,
payloadSummary, status, &ps.SszSignedBeaconBlock, &ps.SszBeaconState, ps.Metrics)
payloadHeader, status, &ps.SszSignedBeaconBlock, &ps.SszBeaconState, ps.Metrics)
if err != nil {
return dw, err
}
@ -477,20 +477,21 @@ func (ps *ProcessSlot) provideFinalHash() (string, string, string, error) {
return blockRoot, stateRoot, eth1DataBlockHash, nil
}
func (ps *ProcessSlot) provideExecutionPayloadDetails() *ExecutionPayloadSummary {
func (ps *ProcessSlot) provideExecutionPayloadDetails() *ExecutionPayloadHeader {
if nil == ps.FullSignedBeaconBlock || !ps.FullSignedBeaconBlock.IsBellatrix() {
return nil
}
payload := ps.FullSignedBeaconBlock.bellatrix.Message.Body.ExecutionPayload
return &ExecutionPayloadSummary{
PayloadBlockNumber: uint64(payload.BlockNumber),
PayloadTimestamp: uint64(payload.Timestamp),
PayloadBlockHash: toHex(payload.BlockHash),
PayloadParentHash: toHex(payload.ParentHash),
PayloadStateRoot: toHex(payload.StateRoot),
PayloadReceiptsRoot: toHex(payload.ReceiptsRoot),
payload := ps.FullSignedBeaconBlock.Block().Body().ExecutionPayloadHeader()
blockNumber := uint64(payload.BlockNumber)
// The earliest blocks on the Bellatrix fork, pre-Merge, have zeroed ExecutionPayloads.
// There is nothing useful to to store in that case, even though the structure exists.
if blockNumber == 0 {
return nil
}
return payload
}
func toHex(r [32]byte) string {