From d6ffa140359d24262903962f145a1236836aeeb4 Mon Sep 17 00:00:00 2001 From: gary rong Date: Tue, 20 Apr 2021 13:27:46 +0800 Subject: [PATCH] core: nuke legacy snapshot supporting (#22663) --- core/blockchain.go | 15 +- core/blockchain_snapshot_test.go | 320 +------------------------------ core/state/snapshot/journal.go | 122 +----------- core/state/snapshot/snapshot.go | 27 --- 4 files changed, 12 insertions(+), 472 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index dfbc5ad0b..49aa1c3e8 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -207,9 +207,8 @@ type BlockChain struct { processor Processor // Block transaction processor interface vmConfig vm.Config - shouldPreserve func(*types.Block) bool // Function used to determine whether should preserve the given block. - terminateInsert func(common.Hash, uint64) bool // Testing hook used to terminate ancient receipt chain insertion. - writeLegacyJournal bool // Testing flag used to flush the snapshot journal in legacy format. + shouldPreserve func(*types.Block) bool // Function used to determine whether should preserve the given block. + terminateInsert func(common.Hash, uint64) bool // Testing hook used to terminate ancient receipt chain insertion. } // NewBlockChain returns a fully initialised block chain using information @@ -1002,14 +1001,8 @@ func (bc *BlockChain) Stop() { var snapBase common.Hash if bc.snaps != nil { var err error - if bc.writeLegacyJournal { - if snapBase, err = bc.snaps.LegacyJournal(bc.CurrentBlock().Root()); err != nil { - log.Error("Failed to journal state snapshot", "err", err) - } - } else { - if snapBase, err = bc.snaps.Journal(bc.CurrentBlock().Root()); err != nil { - log.Error("Failed to journal state snapshot", "err", err) - } + if snapBase, err = bc.snaps.Journal(bc.CurrentBlock().Root()); err != nil { + log.Error("Failed to journal state snapshot", "err", err) } } // Ensure the state of a recent block is also stored to disk before exiting. diff --git a/core/blockchain_snapshot_test.go b/core/blockchain_snapshot_test.go index 70ae31f7d..75c09b421 100644 --- a/core/blockchain_snapshot_test.go +++ b/core/blockchain_snapshot_test.go @@ -39,7 +39,6 @@ import ( // snapshotTestBasic wraps the common testing fields in the snapshot tests. type snapshotTestBasic struct { - legacy bool // Wether write the snapshot journal in legacy format chainBlocks int // Number of blocks to generate for the canonical chain snapshotBlock uint64 // Block number of the relevant snapshot disk layer commitBlock uint64 // Block number for which to commit the state to disk @@ -104,19 +103,13 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo chain.stateCache.TrieDB().Commit(blocks[point-1].Root(), true, nil) } if basic.snapshotBlock > 0 && basic.snapshotBlock == point { - if basic.legacy { - // Here we commit the snapshot disk root to simulate - // committing the legacy snapshot. - rawdb.WriteSnapshotRoot(db, blocks[point-1].Root()) - } else { - // Flushing the entire snap tree into the disk, the - // relavant (a) snapshot root and (b) snapshot generator - // will be persisted atomically. - chain.snaps.Cap(blocks[point-1].Root(), 0) - diskRoot, blockRoot := chain.snaps.DiskRoot(), blocks[point-1].Root() - if !bytes.Equal(diskRoot.Bytes(), blockRoot.Bytes()) { - t.Fatalf("Failed to flush disk layer change, want %x, got %x", blockRoot, diskRoot) - } + // Flushing the entire snap tree into the disk, the + // relavant (a) snapshot root and (b) snapshot generator + // will be persisted atomically. + chain.snaps.Cap(blocks[point-1].Root(), 0) + diskRoot, blockRoot := chain.snaps.DiskRoot(), blocks[point-1].Root() + if !bytes.Equal(diskRoot.Bytes(), blockRoot.Bytes()) { + t.Fatalf("Failed to flush disk layer change, want %x, got %x", blockRoot, diskRoot) } } } @@ -129,12 +122,6 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo basic.db = db basic.gendb = gendb basic.engine = engine - - // Ugly hack, notify the chain to flush the journal in legacy format - // if it's requested. - if basic.legacy { - chain.writeLegacyJournal = true - } return chain, blocks } @@ -484,46 +471,6 @@ func TestRestartWithNewSnapshot(t *testing.T) { // Expected snapshot disk : G test := &snapshotTest{ snapshotTestBasic{ - legacy: false, - chainBlocks: 8, - snapshotBlock: 0, - commitBlock: 0, - expCanonicalBlocks: 8, - expHeadHeader: 8, - expHeadFastBlock: 8, - expHeadBlock: 8, - expSnapshotBottom: 0, // Initial disk layer built from genesis - }, - } - test.test(t) - test.teardown() -} - -// Tests a Geth restart with valid but "legacy" snapshot. Before the shutdown, -// all snapshot journal will be persisted correctly. In this case no snapshot -// recovery is required. -func TestRestartWithLegacySnapshot(t *testing.T) { - // Chain: - // G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD) - // - // Commit: G - // Snapshot: G - // - // SetHead(0) - // - // ------------------------------ - // - // Expected in leveldb: - // G->C1->C2->C3->C4->C5->C6->C7->C8 - // - // Expected head header : C8 - // Expected head fast block: C8 - // Expected head block : C8 - // Expected snapshot disk : G - t.Skip("Legacy format testing is not supported") - test := &snapshotTest{ - snapshotTestBasic{ - legacy: true, chainBlocks: 8, snapshotBlock: 0, commitBlock: 0, @@ -563,7 +510,6 @@ func TestNoCommitCrashWithNewSnapshot(t *testing.T) { // Expected snapshot disk : C4 test := &crashSnapshotTest{ snapshotTestBasic{ - legacy: false, chainBlocks: 8, snapshotBlock: 4, commitBlock: 0, @@ -603,7 +549,6 @@ func TestLowCommitCrashWithNewSnapshot(t *testing.T) { // Expected snapshot disk : C4 test := &crashSnapshotTest{ snapshotTestBasic{ - legacy: false, chainBlocks: 8, snapshotBlock: 4, commitBlock: 2, @@ -643,7 +588,6 @@ func TestHighCommitCrashWithNewSnapshot(t *testing.T) { // Expected snapshot disk : C4 test := &crashSnapshotTest{ snapshotTestBasic{ - legacy: false, chainBlocks: 8, snapshotBlock: 4, commitBlock: 6, @@ -658,131 +602,6 @@ func TestHighCommitCrashWithNewSnapshot(t *testing.T) { test.teardown() } -// Tests a Geth was crashed and restarts with a broken and "legacy format" -// snapshot. In this case the entire legacy snapshot should be discared -// and rebuild from the new chain head. The new head here refers to the -// genesis because there is no committed point. -func TestNoCommitCrashWithLegacySnapshot(t *testing.T) { - // Chain: - // G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD) - // - // Commit: G - // Snapshot: G, C4 - // - // CRASH - // - // ------------------------------ - // - // Expected in leveldb: - // G->C1->C2->C3->C4->C5->C6->C7->C8 - // - // Expected head header : C8 - // Expected head fast block: C8 - // Expected head block : G - // Expected snapshot disk : G - t.Skip("Legacy format testing is not supported") - test := &crashSnapshotTest{ - snapshotTestBasic{ - legacy: true, - chainBlocks: 8, - snapshotBlock: 4, - commitBlock: 0, - expCanonicalBlocks: 8, - expHeadHeader: 8, - expHeadFastBlock: 8, - expHeadBlock: 0, - expSnapshotBottom: 0, // Rebuilt snapshot from the latest HEAD(genesis) - }, - } - test.test(t) - test.teardown() -} - -// Tests a Geth was crashed and restarts with a broken and "legacy format" -// snapshot. In this case the entire legacy snapshot should be discared -// and rebuild from the new chain head. The new head here refers to the -// block-2 because it's committed into the disk. -func TestLowCommitCrashWithLegacySnapshot(t *testing.T) { - // Chain: - // G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD) - // - // Commit: G, C2 - // Snapshot: G, C4 - // - // CRASH - // - // ------------------------------ - // - // Expected in leveldb: - // G->C1->C2->C3->C4->C5->C6->C7->C8 - // - // Expected head header : C8 - // Expected head fast block: C8 - // Expected head block : C2 - // Expected snapshot disk : C2 - t.Skip("Legacy format testing is not supported") - test := &crashSnapshotTest{ - snapshotTestBasic{ - legacy: true, - chainBlocks: 8, - snapshotBlock: 4, - commitBlock: 2, - expCanonicalBlocks: 8, - expHeadHeader: 8, - expHeadFastBlock: 8, - expHeadBlock: 2, - expSnapshotBottom: 2, // Rebuilt snapshot from the latest HEAD - }, - } - test.test(t) - test.teardown() -} - -// Tests a Geth was crashed and restarts with a broken and "legacy format" -// snapshot. In this case the entire legacy snapshot should be discared -// and rebuild from the new chain head. -// -// The new head here refers to the the genesis, the reason is: -// - the state of block-6 is committed into the disk -// - the legacy disk layer of block-4 is committed into the disk -// - the head is rewound the genesis in order to find an available -// state lower than disk layer -func TestHighCommitCrashWithLegacySnapshot(t *testing.T) { - // Chain: - // G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD) - // - // Commit: G, C6 - // Snapshot: G, C4 - // - // CRASH - // - // ------------------------------ - // - // Expected in leveldb: - // G->C1->C2->C3->C4->C5->C6->C7->C8 - // - // Expected head header : C8 - // Expected head fast block: C8 - // Expected head block : G - // Expected snapshot disk : G - t.Skip("Legacy format testing is not supported") - test := &crashSnapshotTest{ - snapshotTestBasic{ - legacy: true, - chainBlocks: 8, - snapshotBlock: 4, - commitBlock: 6, - expCanonicalBlocks: 8, - expHeadHeader: 8, - expHeadFastBlock: 8, - expHeadBlock: 0, - expSnapshotBottom: 0, // Rebuilt snapshot from the latest HEAD(genesis) - }, - } - test.test(t) - test.teardown() -} - // Tests a Geth was running with snapshot enabled. Then restarts without // enabling snapshot and after that re-enable the snapshot again. In this // case the snapshot should be rebuilt with latest chain head. @@ -806,47 +625,6 @@ func TestGappedNewSnapshot(t *testing.T) { // Expected snapshot disk : C10 test := &gappedSnapshotTest{ snapshotTestBasic: snapshotTestBasic{ - legacy: false, - chainBlocks: 8, - snapshotBlock: 0, - commitBlock: 0, - expCanonicalBlocks: 10, - expHeadHeader: 10, - expHeadFastBlock: 10, - expHeadBlock: 10, - expSnapshotBottom: 10, // Rebuilt snapshot from the latest HEAD - }, - gapped: 2, - } - test.test(t) - test.teardown() -} - -// Tests a Geth was running with leagcy snapshot enabled. Then restarts -// without enabling snapshot and after that re-enable the snapshot again. -// In this case the snapshot should be rebuilt with latest chain head. -func TestGappedLegacySnapshot(t *testing.T) { - // Chain: - // G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD) - // - // Commit: G - // Snapshot: G - // - // SetHead(0) - // - // ------------------------------ - // - // Expected in leveldb: - // G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10 - // - // Expected head header : C10 - // Expected head fast block: C10 - // Expected head block : C10 - // Expected snapshot disk : C10 - t.Skip("Legacy format testing is not supported") - test := &gappedSnapshotTest{ - snapshotTestBasic: snapshotTestBasic{ - legacy: true, chainBlocks: 8, snapshotBlock: 0, commitBlock: 0, @@ -885,7 +663,6 @@ func TestSetHeadWithNewSnapshot(t *testing.T) { // Expected snapshot disk : G test := &setHeadSnapshotTest{ snapshotTestBasic: snapshotTestBasic{ - legacy: false, chainBlocks: 8, snapshotBlock: 0, commitBlock: 0, @@ -901,88 +678,6 @@ func TestSetHeadWithNewSnapshot(t *testing.T) { test.teardown() } -// Tests the Geth was running with snapshot(legacy-format) enabled and resetHead -// is applied. In this case the head is rewound to the target(with state available). -// After that the chain is restarted and the original disk layer is kept. -func TestSetHeadWithLegacySnapshot(t *testing.T) { - // Chain: - // G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD) - // - // Commit: G - // Snapshot: G - // - // SetHead(4) - // - // ------------------------------ - // - // Expected in leveldb: - // G->C1->C2->C3->C4 - // - // Expected head header : C4 - // Expected head fast block: C4 - // Expected head block : C4 - // Expected snapshot disk : G - t.Skip("Legacy format testing is not supported") - test := &setHeadSnapshotTest{ - snapshotTestBasic: snapshotTestBasic{ - legacy: true, - chainBlocks: 8, - snapshotBlock: 0, - commitBlock: 0, - expCanonicalBlocks: 4, - expHeadHeader: 4, - expHeadFastBlock: 4, - expHeadBlock: 4, - expSnapshotBottom: 0, // The initial disk layer is built from the genesis - }, - setHead: 4, - } - test.test(t) - test.teardown() -} - -// Tests the Geth was running with snapshot(legacy-format) enabled and upgrades -// the disk layer journal(journal generator) to latest format. After that the Geth -// is restarted from a crash. In this case Geth will find the new-format disk layer -// journal but with legacy-format diff journal(the new-format is never committed), -// and the invalid diff journal is expected to be dropped. -func TestRecoverSnapshotFromCrashWithLegacyDiffJournal(t *testing.T) { - // Chain: - // G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD) - // - // Commit: G - // Snapshot: G - // - // SetHead(0) - // - // ------------------------------ - // - // Expected in leveldb: - // G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10 - // - // Expected head header : C10 - // Expected head fast block: C10 - // Expected head block : C8 - // Expected snapshot disk : C10 - t.Skip("Legacy format testing is not supported") - test := &restartCrashSnapshotTest{ - snapshotTestBasic: snapshotTestBasic{ - legacy: true, - chainBlocks: 8, - snapshotBlock: 0, - commitBlock: 0, - expCanonicalBlocks: 10, - expHeadHeader: 10, - expHeadFastBlock: 10, - expHeadBlock: 8, // The persisted state in the first running - expSnapshotBottom: 10, // The persisted disk layer in the second running - }, - newBlocks: 2, - } - test.test(t) - test.teardown() -} - // Tests the Geth was running with a complete snapshot and then imports a few // more new blocks on top without enabling the snapshot. After the restart, // crash happens. Check everything is ok after the restart. @@ -1006,7 +701,6 @@ func TestRecoverSnapshotFromWipingCrash(t *testing.T) { // Expected snapshot disk : C10 test := &wipeCrashSnapshotTest{ snapshotTestBasic: snapshotTestBasic{ - legacy: false, chainBlocks: 8, snapshotBlock: 4, commitBlock: 0, diff --git a/core/state/snapshot/journal.go b/core/state/snapshot/journal.go index b31e921ca..f8cec4d4e 100644 --- a/core/state/snapshot/journal.go +++ b/core/state/snapshot/journal.go @@ -66,30 +66,6 @@ type journalStorage struct { Vals [][]byte } -// loadAndParseLegacyJournal tries to parse the snapshot journal in legacy format. -func loadAndParseLegacyJournal(db ethdb.KeyValueStore, base *diskLayer) (snapshot, journalGenerator, error) { - // Retrieve the journal, for legacy journal it must exist since even for - // 0 layer it stores whether we've already generated the snapshot or are - // in progress only. - journal := rawdb.ReadSnapshotJournal(db) - if len(journal) == 0 { - return nil, journalGenerator{}, errors.New("missing or corrupted snapshot journal") - } - r := rlp.NewStream(bytes.NewReader(journal), 0) - - // Read the snapshot generation progress for the disk layer - var generator journalGenerator - if err := r.Decode(&generator); err != nil { - return nil, journalGenerator{}, fmt.Errorf("failed to load snapshot progress marker: %v", err) - } - // Load all the snapshot diffs from the journal - snapshot, err := loadDiffLayer(base, r) - if err != nil { - return nil, generator, err - } - return snapshot, generator, nil -} - // loadAndParseJournal tries to parse the snapshot journal in latest format. func loadAndParseJournal(db ethdb.KeyValueStore, base *diskLayer) (snapshot, journalGenerator, error) { // Retrieve the disk layer generator. It must exist, no matter the @@ -163,14 +139,9 @@ func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, cache: fastcache.New(cache * 1024 * 1024), root: baseRoot, } - var legacy bool snapshot, generator, err := loadAndParseJournal(diskdb, base) if err != nil { log.Warn("Failed to load new-format journal", "error", err) - snapshot, generator, err = loadAndParseLegacyJournal(diskdb, base) - legacy = true - } - if err != nil { return nil, err } // Entire snapshot journal loaded, sanity check the head. If the loaded @@ -185,7 +156,7 @@ func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, // If it's legacy snapshot, or it's new-format snapshot but // it's not in recovery mode, returns the error here for // rebuilding the entire snapshot forcibly. - if legacy || !recovery { + if !recovery { return nil, fmt.Errorf("head doesn't match snapshot: have %#x, want %#x", head, root) } // It's in snapshot recovery, the assumption is held that @@ -346,94 +317,3 @@ func (dl *diffLayer) Journal(buffer *bytes.Buffer) (common.Hash, error) { log.Debug("Journalled diff layer", "root", dl.root, "parent", dl.parent.Root()) return base, nil } - -// LegacyJournal writes the persistent layer generator stats into a buffer -// to be stored in the database as the snapshot journal. -// -// Note it's the legacy version which is only used in testing right now. -func (dl *diskLayer) LegacyJournal(buffer *bytes.Buffer) (common.Hash, error) { - // If the snapshot is currently being generated, abort it - var stats *generatorStats - if dl.genAbort != nil { - abort := make(chan *generatorStats) - dl.genAbort <- abort - - if stats = <-abort; stats != nil { - stats.Log("Journalling in-progress snapshot", dl.root, dl.genMarker) - } - } - // Ensure the layer didn't get stale - dl.lock.RLock() - defer dl.lock.RUnlock() - - if dl.stale { - return common.Hash{}, ErrSnapshotStale - } - // Write out the generator marker - entry := journalGenerator{ - Done: dl.genMarker == nil, - Marker: dl.genMarker, - } - if stats != nil { - entry.Accounts = stats.accounts - entry.Slots = stats.slots - entry.Storage = uint64(stats.storage) - } - log.Debug("Legacy journalled disk layer", "root", dl.root) - if err := rlp.Encode(buffer, entry); err != nil { - return common.Hash{}, err - } - return dl.root, nil -} - -// Journal writes the memory layer contents into a buffer to be stored in the -// database as the snapshot journal. -// -// Note it's the legacy version which is only used in testing right now. -func (dl *diffLayer) LegacyJournal(buffer *bytes.Buffer) (common.Hash, error) { - // Journal the parent first - base, err := dl.parent.LegacyJournal(buffer) - if err != nil { - return common.Hash{}, err - } - // Ensure the layer didn't get stale - dl.lock.RLock() - defer dl.lock.RUnlock() - - if dl.Stale() { - return common.Hash{}, ErrSnapshotStale - } - // Everything below was journalled, persist this layer too - if err := rlp.Encode(buffer, dl.root); err != nil { - return common.Hash{}, err - } - destructs := make([]journalDestruct, 0, len(dl.destructSet)) - for hash := range dl.destructSet { - destructs = append(destructs, journalDestruct{Hash: hash}) - } - if err := rlp.Encode(buffer, destructs); err != nil { - return common.Hash{}, err - } - accounts := make([]journalAccount, 0, len(dl.accountData)) - for hash, blob := range dl.accountData { - accounts = append(accounts, journalAccount{Hash: hash, Blob: blob}) - } - if err := rlp.Encode(buffer, accounts); err != nil { - return common.Hash{}, err - } - storage := make([]journalStorage, 0, len(dl.storageData)) - for hash, slots := range dl.storageData { - keys := make([]common.Hash, 0, len(slots)) - vals := make([][]byte, 0, len(slots)) - for key, val := range slots { - keys = append(keys, key) - vals = append(vals, val) - } - storage = append(storage, journalStorage{Hash: hash, Keys: keys, Vals: vals}) - } - if err := rlp.Encode(buffer, storage); err != nil { - return common.Hash{}, err - } - log.Debug("Legacy journalled diff layer", "root", dl.root, "parent", dl.parent.Root()) - return base, nil -} diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index 710ba4d4c..9ecbd4a6c 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -137,10 +137,6 @@ type snapshot interface { // flattening everything down (bad for reorgs). Journal(buffer *bytes.Buffer) (common.Hash, error) - // LegacyJournal is basically identical to Journal. it's the legacy version for - // flushing legacy journal. Now the only purpose of this function is for testing. - LegacyJournal(buffer *bytes.Buffer) (common.Hash, error) - // Stale return whether this layer has become stale (was flattened across) or // if it's still live. Stale() bool @@ -622,29 +618,6 @@ func (t *Tree) Journal(root common.Hash) (common.Hash, error) { return base, nil } -// LegacyJournal is basically identical to Journal. it's the legacy -// version for flushing legacy journal. Now the only purpose of this -// function is for testing. -func (t *Tree) LegacyJournal(root common.Hash) (common.Hash, error) { - // Retrieve the head snapshot to journal from var snap snapshot - snap := t.Snapshot(root) - if snap == nil { - return common.Hash{}, fmt.Errorf("snapshot [%#x] missing", root) - } - // Run the journaling - t.lock.Lock() - defer t.lock.Unlock() - - journal := new(bytes.Buffer) - base, err := snap.(snapshot).LegacyJournal(journal) - if err != nil { - return common.Hash{}, err - } - // Store the journal into the database and return - rawdb.WriteSnapshotJournal(t.diskdb, journal.Bytes()) - return base, nil -} - // Rebuild wipes all available snapshot data from the persistent database and // discard all caches and diff layers. Afterwards, it starts a new snapshot // generator with the given root hash.