From b9c90c5581934454aa71af5b4b668c37b67d92fc Mon Sep 17 00:00:00 2001 From: gary rong Date: Tue, 19 Nov 2019 18:32:57 +0800 Subject: [PATCH] core/rawdb: check hash before return data from ancient db (#20195) * core/rawdb: check hash before return data from ancient db * core/rawdb: fix lint * core/rawdb: calculate the hash in the fly --- core/rawdb/accessors_chain.go | 129 ++++++++++++++++++++--------- core/rawdb/accessors_chain_test.go | 66 +++++++++++++++ 2 files changed, 158 insertions(+), 37 deletions(-) diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index fab7ca56c..4d064a339 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -23,6 +23,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" @@ -173,18 +174,27 @@ func WriteFastTrieProgress(db ethdb.KeyValueWriter, count uint64) { // ReadHeaderRLP retrieves a block header in its raw RLP database encoding. func ReadHeaderRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue { + // First try to look up the data in ancient database. Extra hash + // comparison is necessary since ancient database only maintains + // the canonical data. data, _ := db.Ancient(freezerHeaderTable, number) - if len(data) == 0 { - data, _ = db.Get(headerKey(number, hash)) - // In the background freezer is moving data from leveldb to flatten files. - // So during the first check for ancient db, the data is not yet in there, - // but when we reach into leveldb, the data was already moved. That would - // result in a not found error. - if len(data) == 0 { - data, _ = db.Ancient(freezerHeaderTable, number) - } + if len(data) > 0 && crypto.Keccak256Hash(data) == hash { + return data } - return data + // Then try to look up the data in leveldb. + data, _ = db.Get(headerKey(number, hash)) + if len(data) > 0 { + return data + } + // In the background freezer is moving data from leveldb to flatten files. + // So during the first check for ancient db, the data is not yet in there, + // but when we reach into leveldb, the data was already moved. That would + // result in a not found error. + data, _ = db.Ancient(freezerHeaderTable, number) + if len(data) > 0 && crypto.Keccak256Hash(data) == hash { + return data + } + return nil // Can't find the data anywhere. } // HasHeader verifies the existence of a block header corresponding to the hash. @@ -251,18 +261,33 @@ func deleteHeaderWithoutNumber(db ethdb.KeyValueWriter, hash common.Hash, number // ReadBodyRLP retrieves the block body (transactions and uncles) in RLP encoding. func ReadBodyRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue { + // First try to look up the data in ancient database. Extra hash + // comparison is necessary since ancient database only maintains + // the canonical data. data, _ := db.Ancient(freezerBodiesTable, number) - if len(data) == 0 { - data, _ = db.Get(blockBodyKey(number, hash)) - // In the background freezer is moving data from leveldb to flatten files. - // So during the first check for ancient db, the data is not yet in there, - // but when we reach into leveldb, the data was already moved. That would - // result in a not found error. - if len(data) == 0 { - data, _ = db.Ancient(freezerBodiesTable, number) + if len(data) > 0 { + h, _ := db.Ancient(freezerHashTable, number) + if common.BytesToHash(h) == hash { + return data } } - return data + // Then try to look up the data in leveldb. + data, _ = db.Get(blockBodyKey(number, hash)) + if len(data) > 0 { + return data + } + // In the background freezer is moving data from leveldb to flatten files. + // So during the first check for ancient db, the data is not yet in there, + // but when we reach into leveldb, the data was already moved. That would + // result in a not found error. + data, _ = db.Ancient(freezerBodiesTable, number) + if len(data) > 0 { + h, _ := db.Ancient(freezerHashTable, number) + if common.BytesToHash(h) == hash { + return data + } + } + return nil // Can't find the data anywhere. } // WriteBodyRLP stores an RLP encoded block body into the database. @@ -315,18 +340,33 @@ func DeleteBody(db ethdb.KeyValueWriter, hash common.Hash, number uint64) { // ReadTdRLP retrieves a block's total difficulty corresponding to the hash in RLP encoding. func ReadTdRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue { + // First try to look up the data in ancient database. Extra hash + // comparison is necessary since ancient database only maintains + // the canonical data. data, _ := db.Ancient(freezerDifficultyTable, number) - if len(data) == 0 { - data, _ = db.Get(headerTDKey(number, hash)) - // In the background freezer is moving data from leveldb to flatten files. - // So during the first check for ancient db, the data is not yet in there, - // but when we reach into leveldb, the data was already moved. That would - // result in a not found error. - if len(data) == 0 { - data, _ = db.Ancient(freezerDifficultyTable, number) + if len(data) > 0 { + h, _ := db.Ancient(freezerHashTable, number) + if common.BytesToHash(h) == hash { + return data } } - return data + // Then try to look up the data in leveldb. + data, _ = db.Get(headerTDKey(number, hash)) + if len(data) > 0 { + return data + } + // In the background freezer is moving data from leveldb to flatten files. + // So during the first check for ancient db, the data is not yet in there, + // but when we reach into leveldb, the data was already moved. That would + // result in a not found error. + data, _ = db.Ancient(freezerDifficultyTable, number) + if len(data) > 0 { + h, _ := db.Ancient(freezerHashTable, number) + if common.BytesToHash(h) == hash { + return data + } + } + return nil // Can't find the data anywhere. } // ReadTd retrieves a block's total difficulty corresponding to the hash. @@ -375,18 +415,33 @@ func HasReceipts(db ethdb.Reader, hash common.Hash, number uint64) bool { // ReadReceiptsRLP retrieves all the transaction receipts belonging to a block in RLP encoding. func ReadReceiptsRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue { + // First try to look up the data in ancient database. Extra hash + // comparison is necessary since ancient database only maintains + // the canonical data. data, _ := db.Ancient(freezerReceiptTable, number) - if len(data) == 0 { - data, _ = db.Get(blockReceiptsKey(number, hash)) - // In the background freezer is moving data from leveldb to flatten files. - // So during the first check for ancient db, the data is not yet in there, - // but when we reach into leveldb, the data was already moved. That would - // result in a not found error. - if len(data) == 0 { - data, _ = db.Ancient(freezerReceiptTable, number) + if len(data) > 0 { + h, _ := db.Ancient(freezerHashTable, number) + if common.BytesToHash(h) == hash { + return data } } - return data + // Then try to look up the data in leveldb. + data, _ = db.Get(blockReceiptsKey(number, hash)) + if len(data) > 0 { + return data + } + // In the background freezer is moving data from leveldb to flatten files. + // So during the first check for ancient db, the data is not yet in there, + // but when we reach into leveldb, the data was already moved. That would + // result in a not found error. + data, _ = db.Ancient(freezerReceiptTable, number) + if len(data) > 0 { + h, _ := db.Ancient(freezerHashTable, number) + if common.BytesToHash(h) == hash { + return data + } + } + return nil // Can't find the data anywhere. } // ReadRawReceipts retrieves all the transaction receipts belonging to a block. diff --git a/core/rawdb/accessors_chain_test.go b/core/rawdb/accessors_chain_test.go index 8c8affffd..bb7dad5df 100644 --- a/core/rawdb/accessors_chain_test.go +++ b/core/rawdb/accessors_chain_test.go @@ -20,7 +20,9 @@ import ( "bytes" "encoding/hex" "fmt" + "io/ioutil" "math/big" + "os" "testing" "github.com/ethereum/go-ethereum/common" @@ -358,3 +360,67 @@ func checkReceiptsRLP(have, want types.Receipts) error { } return nil } + +func TestAncientStorage(t *testing.T) { + // Freezer style fast import the chain. + frdir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("failed to create temp freezer dir: %v", err) + } + defer os.Remove(frdir) + + db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "") + if err != nil { + t.Fatalf("failed to create database with ancient backend") + } + // Create a test block + block := types.NewBlockWithHeader(&types.Header{ + Number: big.NewInt(0), + Extra: []byte("test block"), + UncleHash: types.EmptyUncleHash, + TxHash: types.EmptyRootHash, + ReceiptHash: types.EmptyRootHash, + }) + // Ensure nothing non-existent will be read + hash, number := block.Hash(), block.NumberU64() + if blob := ReadHeaderRLP(db, hash, number); len(blob) > 0 { + t.Fatalf("non existent header returned") + } + if blob := ReadBodyRLP(db, hash, number); len(blob) > 0 { + t.Fatalf("non existent body returned") + } + if blob := ReadReceiptsRLP(db, hash, number); len(blob) > 0 { + t.Fatalf("non existent receipts returned") + } + if blob := ReadTdRLP(db, hash, number); len(blob) > 0 { + t.Fatalf("non existent td returned") + } + // Write and verify the header in the database + WriteAncientBlock(db, block, nil, big.NewInt(100)) + if blob := ReadHeaderRLP(db, hash, number); len(blob) == 0 { + t.Fatalf("no header returned") + } + if blob := ReadBodyRLP(db, hash, number); len(blob) == 0 { + t.Fatalf("no body returned") + } + if blob := ReadReceiptsRLP(db, hash, number); len(blob) == 0 { + t.Fatalf("no receipts returned") + } + if blob := ReadTdRLP(db, hash, number); len(blob) == 0 { + t.Fatalf("no td returned") + } + // Use a fake hash for data retrieval, nothing should be returned. + fakeHash := common.BytesToHash([]byte{0x01, 0x02, 0x03}) + if blob := ReadHeaderRLP(db, fakeHash, number); len(blob) != 0 { + t.Fatalf("invalid header returned") + } + if blob := ReadBodyRLP(db, fakeHash, number); len(blob) != 0 { + t.Fatalf("invalid body returned") + } + if blob := ReadReceiptsRLP(db, fakeHash, number); len(blob) != 0 { + t.Fatalf("invalid receipts returned") + } + if blob := ReadTdRLP(db, fakeHash, number); len(blob) != 0 { + t.Fatalf("invalid td returned") + } +}