roysc/plugeth
1
0
Fork 0
forked from cerc-io/plugeth
Code Pull Requests Activity

cmd,internal/era: implement export-history subcommand (#26621)

Browse Source 
* all: implement era format, add history importer/export

* internal/era/e2store: refactor e2store to provide ReadAt interface

* internal/era/e2store: export HeaderSize

* internal/era: refactor era to use ReadAt interface

* internal/era: elevate anonymous func to named

* cmd/utils: don't store entire era file in-memory during import / export

* internal/era: better abstraction between era and e2store

* cmd/era: properly close era files

* cmd/era: don't let defers stack

* cmd/geth: add description for import-history

* cmd/utils: better bytes buffer

* internal/era: error if accumulator has more records than max allowed

* internal/era: better doc comment

* internal/era/e2store: rm superfluous reader, rm superfluous testcases, add fuzzer

* internal/era: avoid some repetition

* internal/era: simplify clauses

* internal/era: unexport things

* internal/era,cmd/utils,cmd/era: change to iterator interface for reading era entries

* cmd/utils: better defer handling in history test

* internal/era,cmd: add number method to era iterator to get the current block number

* internal/era/e2store: avoid double allocation during write

* internal/era,cmd/utils: fix lint issues

* internal/era: add ReaderAt func so entry value can be read lazily

Co-authored-by: lightclient <lightclient@protonmail.com>
Co-authored-by: Martin Holst Swende <martin@swende.se>

* internal/era: improve iterator interface

* internal/era: fix rlp decode of header and correctly read total difficulty

* cmd/era: fix rebase errors

* cmd/era: clearer comments

* cmd,internal: fix comment typos

---------

Co-authored-by: Martin Holst Swende <martin@swende.se>
This commit is contained in:
lightclient
2024-02-07 09:18:27 -07:00
committed by GitHub
co-authored by lightclient Martin Holst Swende
parent 199e0c9ff5
commit 1f50aa7631
15 changed files with 2145 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/era/accumulator.go
+90
View File
@@ -0,0 +1,90 @@
// Copyright 2023 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package era
import (
"fmt"
"math/big"
"github.com/ethereum/go-ethereum/common"
ssz "github.com/ferranbt/fastssz"
)
// ComputeAccumulator calculates the SSZ hash tree root of the Era1
// accumulator of header records.
func ComputeAccumulator(hashes []common.Hash, tds []*big.Int) (common.Hash, error) {
if len(hashes) != len(tds) {
return common.Hash{}, fmt.Errorf("must have equal number hashes as td values")
}
if len(hashes) > MaxEra1Size {
return common.Hash{}, fmt.Errorf("too many records: have %d, max %d", len(hashes), MaxEra1Size)
}
hh := ssz.NewHasher()
for i := range hashes {
rec := headerRecord{hashes[i], tds[i]}
root, err := rec.HashTreeRoot()
if err != nil {
return common.Hash{}, err
}
hh.Append(root[:])
}
hh.MerkleizeWithMixin(0, uint64(len(hashes)), uint64(MaxEra1Size))
return hh.HashRoot()
}
// headerRecord is an individual record for a historical header.
//
// See https://github.com/ethereum/portal-network-specs/blob/master/history-network.md#the-header-accumulator
// for more information.
type headerRecord struct {
Hash common.Hash
TotalDifficulty *big.Int
}
// GetTree completes the ssz.HashRoot interface, but is unused.
func (h *headerRecord) GetTree() (*ssz.Node, error) {
return nil, nil
}
// HashTreeRoot ssz hashes the headerRecord object.
func (h *headerRecord) HashTreeRoot() ([32]byte, error) {
return ssz.HashWithDefaultHasher(h)
}
// HashTreeRootWith ssz hashes the headerRecord object with a hasher.
func (h *headerRecord) HashTreeRootWith(hh ssz.HashWalker) (err error) {
hh.PutBytes(h.Hash[:])
td := bigToBytes32(h.TotalDifficulty)
hh.PutBytes(td[:])
hh.Merkleize(0)
return
}
// bigToBytes32 converts a big.Int into a little-endian 32-byte array.
func bigToBytes32(n *big.Int) (b [32]byte) {
n.FillBytes(b[:])
reverseOrder(b[:])
return
}
// reverseOrder reverses the byte order of a slice.
func reverseOrder(b []byte) []byte {
for i := 0; i < 16; i++ {
b[i], b[32-i-1] = b[32-i-1], b[i]
}
return b
}
internal/era/builder.go
+228
View File
@@ -0,0 +1,228 @@
// Copyright 2023 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package era
import (
"bytes"
"encoding/binary"
"fmt"
"io"
"math/big"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/internal/era/e2store"
"github.com/ethereum/go-ethereum/rlp"
"github.com/golang/snappy"
)
// Builder is used to create Era1 archives of block data.
//
// Era1 files are themselves e2store files. For more information on this format,
// see https://github.com/status-im/nimbus-eth2/blob/stable/docs/e2store.md.
//
// The overall structure of an Era1 file follows closely the structure of an Era file
// which contains consensus Layer data (and as a byproduct, EL data after the merge).
//
// The structure can be summarized through this definition:
//
// era1 := Version | block-tuple* | other-entries* | Accumulator | BlockIndex
// block-tuple := CompressedHeader | CompressedBody | CompressedReceipts | TotalDifficulty
//
// Each basic element is its own entry:
//
// Version = { type: [0x65, 0x32], data: nil }
// CompressedHeader = { type: [0x03, 0x00], data: snappyFramed(rlp(header)) }
// CompressedBody = { type: [0x04, 0x00], data: snappyFramed(rlp(body)) }
// CompressedReceipts = { type: [0x05, 0x00], data: snappyFramed(rlp(receipts)) }
// TotalDifficulty = { type: [0x06, 0x00], data: uint256(header.total_difficulty) }
// Accumulator = { type: [0x07, 0x00], data: accumulator-root }
// BlockIndex = { type: [0x32, 0x66], data: block-index }
//
// Accumulator is computed by constructing an SSZ list of header-records of length at most
// 8192 and then calculating the hash_tree_root of that list.
//
// header-record := { block-hash: Bytes32, total-difficulty: Uint256 }
// accumulator := hash_tree_root([]header-record, 8192)
//
// BlockIndex stores relative offsets to each compressed block entry. The
// format is:
//
// block-index := starting-number | index | index | index ... | count
//
// starting-number is the first block number in the archive. Every index is a
// defined relative to index's location in the file. The total number of block
// entries in the file is recorded in count.
//
// Due to the accumulator size limit of 8192, the maximum number of blocks in
// an Era1 batch is also 8192.
type Builder struct {
w *e2store.Writer
startNum *uint64
startTd *big.Int
indexes []uint64
hashes []common.Hash
tds []*big.Int
written int
buf *bytes.Buffer
snappy *snappy.Writer
}
// NewBuilder returns a new Builder instance.
func NewBuilder(w io.Writer) *Builder {
buf := bytes.NewBuffer(nil)
return &Builder{
w: e2store.NewWriter(w),
buf: buf,
snappy: snappy.NewBufferedWriter(buf),
}
}
// Add writes a compressed block entry and compressed receipts entry to the
// underlying e2store file.
func (b *Builder) Add(block *types.Block, receipts types.Receipts, td *big.Int) error {
eh, err := rlp.EncodeToBytes(block.Header())
if err != nil {
return err
}
eb, err := rlp.EncodeToBytes(block.Body())
if err != nil {
return err
}
er, err := rlp.EncodeToBytes(receipts)
if err != nil {
return err
}
return b.AddRLP(eh, eb, er, block.NumberU64(), block.Hash(), td, block.Difficulty())
}
// AddRLP writes a compressed block entry and compressed receipts entry to the
// underlying e2store file.
func (b *Builder) AddRLP(header, body, receipts []byte, number uint64, hash common.Hash, td, difficulty *big.Int) error {
// Write Era1 version entry before first block.
if b.startNum == nil {
if err := writeVersion(b.w); err != nil {
return err
}
n := number
b.startNum = &n
b.startTd = new(big.Int).Sub(td, difficulty)
}
if len(b.indexes) >= MaxEra1Size {
return fmt.Errorf("exceeds maximum batch size of %d", MaxEra1Size)
}
b.indexes = append(b.indexes, uint64(b.written))
b.hashes = append(b.hashes, hash)
b.tds = append(b.tds, td)
// Write block data.
if err := b.snappyWrite(TypeCompressedHeader, header); err != nil {
return err
}
if err := b.snappyWrite(TypeCompressedBody, body); err != nil {
return err
}
if err := b.snappyWrite(TypeCompressedReceipts, receipts); err != nil {
return err
}
// Also write total difficulty, but don't snappy encode.
btd := bigToBytes32(td)
n, err := b.w.Write(TypeTotalDifficulty, btd[:])
b.written += n
if err != nil {
return err
}
return nil
}
// Finalize computes the accumulator and block index values, then writes the
// corresponding e2store entries.
func (b *Builder) Finalize() (common.Hash, error) {
if b.startNum == nil {
return common.Hash{}, fmt.Errorf("finalize called on empty builder")
}
// Compute accumulator root and write entry.
root, err := ComputeAccumulator(b.hashes, b.tds)
if err != nil {
return common.Hash{}, fmt.Errorf("error calculating accumulator root: %w", err)
}
n, err := b.w.Write(TypeAccumulator, root[:])
b.written += n
if err != nil {
return common.Hash{}, fmt.Errorf("error writing accumulator: %w", err)
}
// Get beginning of index entry to calculate block relative offset.
base := int64(b.written + (3 * 8)) // skip e2store header (type, length) and start block
// Construct block index. Detailed format described in Builder
// documentation, but it is essentially encoded as:
// "start | index | index | ... | count"
var (
count = len(b.indexes)
index = make([]byte, 16+count*8)
)
binary.LittleEndian.PutUint64(index, *b.startNum)
// Each offset is relative from the position it is encoded in the
// index. This means that even if the same block was to be included in
// the index twice (this would be invalid anyways), the relative offset
// would be different. The idea with this is that after reading a
// relative offset, the corresponding block can be quickly read by
// performing a seek relative to the current position.
for i, offset := range b.indexes {
relative := int64(offset) - (base + int64(i)*8)
binary.LittleEndian.PutUint64(index[8+i*8:], uint64(relative))
}
binary.LittleEndian.PutUint64(index[8+count*8:], uint64(count))
// Finally, write the block index entry.
if _, err := b.w.Write(TypeBlockIndex, index); err != nil {
return common.Hash{}, fmt.Errorf("unable to write block index: %w", err)
}
return root, nil
}
// snappyWrite is a small helper to take care snappy encoding and writing an e2store entry.
func (b *Builder) snappyWrite(typ uint16, in []byte) error {
var (
buf = b.buf
s = b.snappy
)
buf.Reset()
s.Reset(buf)
if _, err := b.snappy.Write(in); err != nil {
return fmt.Errorf("error snappy encoding: %w", err)
}
if err := s.Flush(); err != nil {
return fmt.Errorf("error flushing snappy encoding: %w", err)
}
n, err := b.w.Write(typ, b.buf.Bytes())
b.written += n
if err != nil {
return fmt.Errorf("error writing e2store entry: %w", err)
}
return nil
}
// writeVersion writes a version entry to e2store.
func writeVersion(w *e2store.Writer) error {
_, err := w.Write(TypeVersion, nil)
return err
}
internal/era/e2store/e2store.go
+220
View File
@@ -0,0 +1,220 @@
// Copyright 2023 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package e2store
import (
"encoding/binary"
"fmt"
"io"
)
const (
headerSize = 8
valueSizeLimit = 1024 * 1024 * 50
)
// Entry is a variable-length-data record in an e2store.
type Entry struct {
Type uint16
Value []byte
}
// Writer writes entries using e2store encoding.
// For more information on this format, see:
// https://github.com/status-im/nimbus-eth2/blob/stable/docs/e2store.md
type Writer struct {
w io.Writer
}
// NewWriter returns a new Writer that writes to w.
func NewWriter(w io.Writer) *Writer {
return &Writer{w}
}
// Write writes a single e2store entry to w.
// An entry is encoded in a type-length-value format. The first 8 bytes of the
// record store the type (2 bytes), the length (4 bytes), and some reserved
// data (2 bytes). The remaining bytes store b.
func (w *Writer) Write(typ uint16, b []byte) (int, error) {
buf := make([]byte, headerSize)
binary.LittleEndian.PutUint16(buf, typ)
binary.LittleEndian.PutUint32(buf[2:], uint32(len(b)))
// Write header.
if n, err := w.w.Write(buf); err != nil {
return n, err
}
// Write value, return combined write size.
n, err := w.w.Write(b)
return n + headerSize, err
}
// A Reader reads entries from an e2store-encoded file.
// For more information on this format, see
// https://github.com/status-im/nimbus-eth2/blob/stable/docs/e2store.md
type Reader struct {
r io.ReaderAt
offset int64
}
// NewReader returns a new Reader that reads from r.
func NewReader(r io.ReaderAt) *Reader {
return &Reader{r, 0}
}
// Read reads one Entry from r.
func (r *Reader) Read() (*Entry, error) {
var e Entry
n, err := r.ReadAt(&e, r.offset)
if err != nil {
return nil, err
}
r.offset += int64(n)
return &e, nil
}
// ReadAt reads one Entry from r at the specified offset.
func (r *Reader) ReadAt(entry *Entry, off int64) (int, error) {
typ, length, err := r.ReadMetadataAt(off)
if err != nil {
return 0, err
}
entry.Type = typ
// Check length bounds.
if length > valueSizeLimit {
return headerSize, fmt.Errorf("item larger than item size limit %d: have %d", valueSizeLimit, length)
}
if length == 0 {
return headerSize, nil
}
// Read value.
val := make([]byte, length)
if n, err := r.r.ReadAt(val, off+headerSize); err != nil {
n += headerSize
// An entry with a non-zero length should not return EOF when
// reading the value.
if err == io.EOF {
return n, io.ErrUnexpectedEOF
}
return n, err
}
entry.Value = val
return int(headerSize + length), nil
}
// ReaderAt returns an io.Reader delivering value data for the entry at
// the specified offset. If the entry type does not match the expected type, an
// error is returned.
func (r *Reader) ReaderAt(expectedType uint16, off int64) (io.Reader, int, error) {
// problem = need to return length+headerSize not just value length via section reader
typ, length, err := r.ReadMetadataAt(off)
if err != nil {
return nil, headerSize, err
}
if typ != expectedType {
return nil, headerSize, fmt.Errorf("wrong type, want %d have %d", expectedType, typ)
}
if length > valueSizeLimit {
return nil, headerSize, fmt.Errorf("item larger than item size limit %d: have %d", valueSizeLimit, length)
}
return io.NewSectionReader(r.r, off+headerSize, int64(length)), headerSize + int(length), nil
}
// LengthAt reads the header at off and returns the total length of the entry,
// including header.
func (r *Reader) LengthAt(off int64) (int64, error) {
_, length, err := r.ReadMetadataAt(off)
if err != nil {
return 0, err
}
return int64(length) + headerSize, nil
}
// ReadMetadataAt reads the header metadata at the given offset.
func (r *Reader) ReadMetadataAt(off int64) (typ uint16, length uint32, err error) {
b := make([]byte, headerSize)
if n, err := r.r.ReadAt(b, off); err != nil {
if err == io.EOF && n > 0 {
return 0, 0, io.ErrUnexpectedEOF
}
return 0, 0, err
}
typ = binary.LittleEndian.Uint16(b)
length = binary.LittleEndian.Uint32(b[2:])
// Check reserved bytes of header.
if b[6] != 0 || b[7] != 0 {
return 0, 0, fmt.Errorf("reserved bytes are non-zero")
}
return typ, length, nil
}
// Find returns the first entry with the matching type.
func (r *Reader) Find(want uint16) (*Entry, error) {
var (
off int64
typ uint16
length uint32
err error
)
for {
typ, length, err = r.ReadMetadataAt(off)
if err == io.EOF {
return nil, io.EOF
} else if err != nil {
return nil, err
}
if typ == want {
var e Entry
if _, err := r.ReadAt(&e, off); err != nil {
return nil, err
}
return &e, nil
}
off += int64(headerSize + length)
}
}
// FindAll returns all entries with the matching type.
func (r *Reader) FindAll(want uint16) ([]*Entry, error) {
var (
off int64
typ uint16
length uint32
entries []*Entry
err error
)
for {
typ, length, err = r.ReadMetadataAt(off)
if err == io.EOF {
return entries, nil
} else if err != nil {
return entries, err
}
if typ == want {
e := new(Entry)
if _, err := r.ReadAt(e, off); err != nil {
return entries, err
}
entries = append(entries, e)
}
off += int64(headerSize + length)
}
}
internal/era/e2store/e2store_test.go
+150
View File
@@ -0,0 +1,150 @@
// Copyright 2023 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package e2store
import (
"bytes"
"fmt"
"io"
"testing"
"github.com/ethereum/go-ethereum/common"
)
func TestEncode(t *testing.T) {
for _, test := range []struct {
entries []Entry
want string
name string
}{
{
name: "emptyEntry",
entries: []Entry{{0xffff, nil}},
want: "ffff000000000000",
},
{
name: "beef",
entries: []Entry{{42, common.Hex2Bytes("beef")}},
want: "2a00020000000000beef",
},
{
name: "twoEntries",
entries: []Entry{
{42, common.Hex2Bytes("beef")},
{9, common.Hex2Bytes("abcdabcd")},
},
want: "2a00020000000000beef0900040000000000abcdabcd",
},
} {
tt := test
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
var (
b = bytes.NewBuffer(nil)
w = NewWriter(b)
)
for _, e := range tt.entries {
if _, err := w.Write(e.Type, e.Value); err != nil {
t.Fatalf("encoding error: %v", err)
}
}
if want, have := common.FromHex(tt.want), b.Bytes(); !bytes.Equal(want, have) {
t.Fatalf("encoding mismatch (want %x, have %x", want, have)
}
r := NewReader(bytes.NewReader(b.Bytes()))
for _, want := range tt.entries {
have, err := r.Read()
if err != nil {
t.Fatalf("decoding error: %v", err)
}
if have.Type != want.Type {
t.Fatalf("decoded entry does type mismatch (want %v, got %v)", want.Type, have.Type)
}
if !bytes.Equal(have.Value, want.Value) {
t.Fatalf("decoded entry does not match (want %#x, got %#x)", want.Value, have.Value)
}
}
})
}
}
func TestDecode(t *testing.T) {
for i, tt := range []struct {
have string
err error
}{
{ // basic valid decoding
have: "ffff000000000000",
},
{ // basic invalid decoding
have: "ffff000000000001",
err: fmt.Errorf("reserved bytes are non-zero"),
},
{ // no more entries to read, returns EOF
have: "",
err: io.EOF,
},
{ // malformed type
have: "bad",
err: io.ErrUnexpectedEOF,
},
{ // malformed length
have: "badbeef",
err: io.ErrUnexpectedEOF,
},
{ // specified length longer than actual value
have: "beef010000000000",
err: io.ErrUnexpectedEOF,
},
} {
r := NewReader(bytes.NewReader(common.FromHex(tt.have)))
if tt.err != nil {
_, err := r.Read()
if err == nil && tt.err != nil {
t.Fatalf("test %d, expected error, got none", i)
}
if err != nil && tt.err == nil {
t.Fatalf("test %d, expected no error, got %v", i, err)
}
if err != nil && tt.err != nil && err.Error() != tt.err.Error() {
t.Fatalf("expected error %v, got %v", tt.err, err)
}
continue
}
}
}
func FuzzCodec(f *testing.F) {
f.Fuzz(func(t *testing.T, input []byte) {
r := NewReader(bytes.NewReader(input))
entry, err := r.Read()
if err != nil {
return
}
var (
b = bytes.NewBuffer(nil)
w = NewWriter(b)
)
w.Write(entry.Type, entry.Value)
output := b.Bytes()
// Only care about the input that was actually consumed
input = input[:r.offset]
if !bytes.Equal(input, output) {
t.Fatalf("decode-encode mismatch, input %#x output %#x", input, output)
}
})
}
internal/era/era.go
+282
View File
@@ -0,0 +1,282 @@
// Copyright 2023 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package era
import (
"encoding/binary"
"fmt"
"io"
"math/big"
"os"
"path"
"strconv"
"strings"
"sync"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/internal/era/e2store"
"github.com/ethereum/go-ethereum/rlp"
"github.com/golang/snappy"
)
var (
TypeVersion uint16 = 0x3265
TypeCompressedHeader uint16 = 0x03
TypeCompressedBody uint16 = 0x04
TypeCompressedReceipts uint16 = 0x05
TypeTotalDifficulty uint16 = 0x06
TypeAccumulator uint16 = 0x07
TypeBlockIndex uint16 = 0x3266
MaxEra1Size = 8192
)
// Filename returns a recognizable Era1-formatted file name for the specified
// epoch and network.
func Filename(network string, epoch int, root common.Hash) string {
return fmt.Sprintf("%s-%05d-%s.era1", network, epoch, root.Hex()[2:10])
}
// ReadDir reads all the era1 files in a directory for a given network.
// Format: <network>-<epoch>-<hexroot>.era1
func ReadDir(dir, network string) ([]string, error) {
entries, err := os.ReadDir(dir)
if err != nil {
return nil, fmt.Errorf("error reading directory %s: %w", dir, err)
}
var (
next = uint64(0)
eras []string
)
for _, entry := range entries {
if path.Ext(entry.Name()) != ".era1" {
continue
}
parts := strings.Split(entry.Name(), "-")
if len(parts) != 3 || parts[0] != network {
// invalid era1 filename, skip
continue
}
if epoch, err := strconv.ParseUint(parts[1], 10, 64); err != nil {
return nil, fmt.Errorf("malformed era1 filename: %s", entry.Name())
} else if epoch != next {
return nil, fmt.Errorf("missing epoch %d", next)
}
next += 1
eras = append(eras, entry.Name())
}
return eras, nil
}
type ReadAtSeekCloser interface {
io.ReaderAt
io.Seeker
io.Closer
}
// Era reads and Era1 file.
type Era struct {
f ReadAtSeekCloser // backing era1 file
s *e2store.Reader // e2store reader over f
m metadata // start, count, length info
mu *sync.Mutex // lock for buf
buf [8]byte // buffer reading entry offsets
}
// From returns an Era backed by f.
func From(f ReadAtSeekCloser) (*Era, error) {
m, err := readMetadata(f)
if err != nil {
return nil, err
}
return &Era{
f: f,
s: e2store.NewReader(f),
m: m,
mu: new(sync.Mutex),
}, nil
}
// Open returns an Era backed by the given filename.
func Open(filename string) (*Era, error) {
f, err := os.Open(filename)
if err != nil {
return nil, err
}
return From(f)
}
func (e *Era) Close() error {
return e.f.Close()
}
func (e *Era) GetBlockByNumber(num uint64) (*types.Block, error) {
if e.m.start > num || e.m.start+e.m.count <= num {
return nil, fmt.Errorf("out-of-bounds")
}
off, err := e.readOffset(num)
if err != nil {
return nil, err
}
r, n, err := newSnappyReader(e.s, TypeCompressedHeader, off)
if err != nil {
return nil, err
}
var header types.Header
if err := rlp.Decode(r, &header); err != nil {
return nil, err
}
off += n
r, _, err = newSnappyReader(e.s, TypeCompressedBody, off)
if err != nil {
return nil, err
}
var body types.Body
if err := rlp.Decode(r, &body); err != nil {
return nil, err
}
return types.NewBlockWithHeader(&header).WithBody(body.Transactions, body.Uncles), nil
}
// Accumulator reads the accumulator entry in the Era1 file.
func (e *Era) Accumulator() (common.Hash, error) {
entry, err := e.s.Find(TypeAccumulator)
if err != nil {
return common.Hash{}, err
}
return common.BytesToHash(entry.Value), nil
}
// InitialTD returns initial total difficulty before the difficulty of the
// first block of the Era1 is applied.
func (e *Era) InitialTD() (*big.Int, error) {
var (
r io.Reader
header types.Header
rawTd []byte
n int64
off int64
err error
)
// Read first header.
if off, err = e.readOffset(e.m.start); err != nil {
return nil, err
}
if r, n, err = newSnappyReader(e.s, TypeCompressedHeader, off); err != nil {
return nil, err
}
if err := rlp.Decode(r, &header); err != nil {
return nil, err
}
off += n
// Skip over next two records.
for i := 0; i < 2; i++ {
length, err := e.s.LengthAt(off)
if err != nil {
return nil, err
}
off += length
}
// Read total difficulty after first block.
if r, _, err = e.s.ReaderAt(TypeTotalDifficulty, off); err != nil {
return nil, err
}
rawTd, err = io.ReadAll(r)
if err != nil {
return nil, err
}
td := new(big.Int).SetBytes(reverseOrder(rawTd))
return td.Sub(td, header.Difficulty), nil
}
// Start returns the listed start block.
func (e *Era) Start() uint64 {
return e.m.start
}
// Count returns the total number of blocks in the Era1.
func (e *Era) Count() uint64 {
return e.m.count
}
// readOffset reads a specific block's offset from the block index. The value n
// is the absolute block number desired.
func (e *Era) readOffset(n uint64) (int64, error) {
var (
firstIndex = -8 - int64(e.m.count)*8 // size of count - index entries
indexOffset = int64(n-e.m.start) * 8 // desired index * size of indexes
offOffset = e.m.length + firstIndex + indexOffset // offset of block offset
)
e.mu.Lock()
defer e.mu.Unlock()
clearBuffer(e.buf[:])
if _, err := e.f.ReadAt(e.buf[:], offOffset); err != nil {
return 0, err
}
// Since the block offset is relative from its location + size of index
// value (8), we need to add it to it's offset to get the block's
// absolute offset.
return offOffset + 8 + int64(binary.LittleEndian.Uint64(e.buf[:])), nil
}
// newReader returns a snappy.Reader for the e2store entry value at off.
func newSnappyReader(e *e2store.Reader, expectedType uint16, off int64) (io.Reader, int64, error) {
r, n, err := e.ReaderAt(expectedType, off)
if err != nil {
return nil, 0, err
}
return snappy.NewReader(r), int64(n), err
}
// clearBuffer zeroes out the buffer.
func clearBuffer(buf []byte) {
for i := 0; i < len(buf); i++ {
buf[i] = 0
}
}
// metadata wraps the metadata in the block index.
type metadata struct {
start uint64
count uint64
length int64
}
// readMetadata reads the metadata stored in an Era1 file's block index.
func readMetadata(f ReadAtSeekCloser) (m metadata, err error) {
// Determine length of reader.
if m.length, err = f.Seek(0, io.SeekEnd); err != nil {
return
}
b := make([]byte, 16)
// Read count. It's the last 8 bytes of the file.
if _, err = f.ReadAt(b[:8], m.length-8); err != nil {
return
}
m.count = binary.LittleEndian.Uint64(b)
// Read start. It's at the offset -sizeof(m.count) -
// count*sizeof(indexEntry) - sizeof(m.start)
if _, err = f.ReadAt(b[8:], m.length-16-int64(m.count*8)); err != nil {
return
}
m.start = binary.LittleEndian.Uint64(b[8:])
return
}
internal/era/era_test.go
+142
View File
@@ -0,0 +1,142 @@
// Copyright 2023 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package era
import (
"bytes"
"io"
"math/big"
"os"
"testing"
"github.com/ethereum/go-ethereum/common"
)
type testchain struct {
headers [][]byte
bodies [][]byte
receipts [][]byte
tds []*big.Int
}
func TestEra1Builder(t *testing.T) {
// Get temp directory.
f, err := os.CreateTemp("", "era1-test")
if err != nil {
t.Fatalf("error creating temp file: %v", err)
}
defer f.Close()
var (
builder = NewBuilder(f)
chain = testchain{}
)
for i := 0; i < 128; i++ {
chain.headers = append(chain.headers, []byte{byte('h'), byte(i)})
chain.bodies = append(chain.bodies, []byte{byte('b'), byte(i)})
chain.receipts = append(chain.receipts, []byte{byte('r'), byte(i)})
chain.tds = append(chain.tds, big.NewInt(int64(i)))
}
// Write blocks to Era1.
for i := 0; i < len(chain.headers); i++ {
var (
header = chain.headers[i]
body = chain.bodies[i]
receipts = chain.receipts[i]
hash = common.Hash{byte(i)}
td = chain.tds[i]
)
if err = builder.AddRLP(header, body, receipts, uint64(i), hash, td, big.NewInt(1)); err != nil {
t.Fatalf("error adding entry: %v", err)
}
}
// Finalize Era1.
if _, err := builder.Finalize(); err != nil {
t.Fatalf("error finalizing era1: %v", err)
}
// Verify Era1 contents.
e, err := Open(f.Name())
if err != nil {
t.Fatalf("failed to open era: %v", err)
}
it, err := NewRawIterator(e)
if err != nil {
t.Fatalf("failed to make iterator: %s", err)
}
for i := uint64(0); i < uint64(len(chain.headers)); i++ {
if !it.Next() {
t.Fatalf("expected more entries")
}
if it.Error() != nil {
t.Fatalf("unexpected error %v", it.Error())
}
// Check headers.
header, err := io.ReadAll(it.Header)
if err != nil {
t.Fatalf("error reading header: %v", err)
}
if !bytes.Equal(header, chain.headers[i]) {
t.Fatalf("mismatched header: want %s, got %s", chain.headers[i], header)
}
// Check bodies.
body, err := io.ReadAll(it.Body)
if err != nil {
t.Fatalf("error reading body: %v", err)
}
if !bytes.Equal(body, chain.bodies[i]) {
t.Fatalf("mismatched body: want %s, got %s", chain.bodies[i], body)
}
// Check receipts.
receipts, err := io.ReadAll(it.Receipts)
if err != nil {
t.Fatalf("error reading receipts: %v", err)
}
if !bytes.Equal(receipts, chain.receipts[i]) {
t.Fatalf("mismatched receipts: want %s, got %s", chain.receipts[i], receipts)
}
// Check total difficulty.
rawTd, err := io.ReadAll(it.TotalDifficulty)
if err != nil {
t.Fatalf("error reading td: %v", err)
}
td := new(big.Int).SetBytes(reverseOrder(rawTd))
if td.Cmp(chain.tds[i]) != 0 {
t.Fatalf("mismatched tds: want %s, got %s", chain.tds[i], td)
}
}
}
func TestEraFilename(t *testing.T) {
for i, tt := range []struct {
network string
epoch int
root common.Hash
expected string
}{
{"mainnet", 1, common.Hash{1}, "mainnet-00001-01000000.era1"},
{"goerli", 99999, common.HexToHash("0xdeadbeef00000000000000000000000000000000000000000000000000000000"), "goerli-99999-deadbeef.era1"},
} {
got := Filename(tt.network, tt.epoch, tt.root)
if tt.expected != got {
t.Errorf("test %d: invalid filename: want %s, got %s", i, tt.expected, got)
}
}
}
internal/era/iterator.go
+197
View File
@@ -0,0 +1,197 @@
// Copyright 2023 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package era
import (
"fmt"
"io"
"math/big"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/rlp"
)
// Iterator wraps RawIterator and returns decoded Era1 entries.
type Iterator struct {
inner *RawIterator
}
// NewRawIterator returns a new Iterator instance. Next must be immediately
// called on new iterators to load the first item.
func NewIterator(e *Era) (*Iterator, error) {
inner, err := NewRawIterator(e)
if err != nil {
return nil, err
}
return &Iterator{inner}, nil
}
// Next moves the iterator to the next block entry. It returns false when all
// items have been read or an error has halted its progress. Block, Receipts,
// and BlockAndReceipts should no longer be called after false is returned.
func (it *Iterator) Next() bool {
return it.inner.Next()
}
// Number returns the current number block the iterator will return.
func (it *Iterator) Number() uint64 {
return it.inner.next - 1
}
// Error returns the error status of the iterator. It should be called before
// reading from any of the iterator's values.
func (it *Iterator) Error() error {
return it.inner.Error()
}
// Block returns the block for the iterator's current position.
func (it *Iterator) Block() (*types.Block, error) {
if it.inner.Header == nil || it.inner.Body == nil {
return nil, fmt.Errorf("header and body must be non-nil")
}
var (
header types.Header
body types.Body
)
if err := rlp.Decode(it.inner.Header, &header); err != nil {
return nil, err
}
if err := rlp.Decode(it.inner.Body, &body); err != nil {
return nil, err
}
return types.NewBlockWithHeader(&header).WithBody(body.Transactions, body.Uncles), nil
}
// Receipts returns the receipts for the iterator's current position.
func (it *Iterator) Receipts() (types.Receipts, error) {
if it.inner.Receipts == nil {
return nil, fmt.Errorf("receipts must be non-nil")
}
var receipts types.Receipts
err := rlp.Decode(it.inner.Receipts, &receipts)
return receipts, err
}
// BlockAndReceipts returns the block and receipts for the iterator's current
// position.
func (it *Iterator) BlockAndReceipts() (*types.Block, types.Receipts, error) {
b, err := it.Block()
if err != nil {
return nil, nil, err
}
r, err := it.Receipts()
if err != nil {
return nil, nil, err
}
return b, r, nil
}
// TotalDifficulty returns the total difficulty for the iterator's current
// position.
func (it *Iterator) TotalDifficulty() (*big.Int, error) {
td, err := io.ReadAll(it.inner.TotalDifficulty)
if err != nil {
return nil, err
}
return new(big.Int).SetBytes(reverseOrder(td)), nil
}
// RawIterator reads an RLP-encode Era1 entries.
type RawIterator struct {
e *Era // backing Era1
next uint64 // next block to read
err error // last error
Header io.Reader
Body io.Reader
Receipts io.Reader
TotalDifficulty io.Reader
}
// NewRawIterator returns a new RawIterator instance. Next must be immediately
// called on new iterators to load the first item.
func NewRawIterator(e *Era) (*RawIterator, error) {
return &RawIterator{
e: e,
next: e.m.start,
}, nil
}
// Next moves the iterator to the next block entry. It returns false when all
// items have been read or an error has halted its progress. Header, Body,
// Receipts, TotalDifficulty will be set to nil in the case returning false or
// finding an error and should therefore no longer be read from.
func (it *RawIterator) Next() bool {
// Clear old errors.
it.err = nil
if it.e.m.start+it.e.m.count <= it.next {
it.clear()
return false
}
off, err := it.e.readOffset(it.next)
if err != nil {
// Error here means block index is corrupted, so don't
// continue.
it.clear()
it.err = err
return false
}
var n int64
if it.Header, n, it.err = newSnappyReader(it.e.s, TypeCompressedHeader, off); it.err != nil {
it.clear()
return true
}
off += n
if it.Body, n, it.err = newSnappyReader(it.e.s, TypeCompressedBody, off); it.err != nil {
it.clear()
return true
}
off += n
if it.Receipts, n, it.err = newSnappyReader(it.e.s, TypeCompressedReceipts, off); it.err != nil {
it.clear()
return true
}
off += n
if it.TotalDifficulty, _, it.err = it.e.s.ReaderAt(TypeTotalDifficulty, off); it.err != nil {
it.clear()
return true
}
it.next += 1
return true
}
// Number returns the current number block the iterator will return.
func (it *RawIterator) Number() uint64 {
return it.next - 1
}
// Error returns the error status of the iterator. It should be called before
// reading from any of the iterator's values.
func (it *RawIterator) Error() error {
if it.err == io.EOF {
return nil
}
return it.err
}
// clear sets all the outputs to nil.
func (it *RawIterator) clear() {
it.Header = nil
it.Body = nil
it.Receipts = nil
it.TotalDifficulty = nil
}