Merge tag 'v1.10.13' into HEAD

core/rawdb/accessors_chain.go

@@ -669,7 +669,7 @@ func deriveLogFields(receipts []*receiptLogs, hash common.Hash, number uint64, t
 // ReadLogs retrieves the logs for all transactions in a block. The log fields
 // are populated with metadata. In case the receipts or the block body
 // are not found, a nil is returned.
-func ReadLogs(db ethdb.Reader, hash common.Hash, number uint64) [][]*types.Log {
+func ReadLogs(db ethdb.Reader, hash common.Hash, number uint64, config *params.ChainConfig) [][]*types.Log {
 	// Retrieve the flattened receipt slice
 	data := ReadReceiptsRLP(db, hash, number)
 	if len(data) == 0 {
@@ -677,7 +677,12 @@ func ReadLogs(db ethdb.Reader, hash common.Hash, number uint64) [][]*types.Log {
 	}
 	receipts := []*receiptLogs{}
 	if err := rlp.DecodeBytes(data, &receipts); err != nil {
-		log.Error("Invalid receipt array RLP", "hash", hash, "err", err)
+		// Receipts might be in the legacy format, try decoding that.
+		// TODO: to be removed after users migrated
+		if logs := readLegacyLogs(db, hash, number, config); logs != nil {
+			return logs
+		}
+		log.Error("Invalid receipt array RLP", "hash", "err", err)
 		return nil
 	}
 
@@ -697,6 +702,21 @@ func ReadLogs(db ethdb.Reader, hash common.Hash, number uint64) [][]*types.Log {
 	return logs
 }
 
+// readLegacyLogs is a temporary workaround for when trying to read logs
+// from a block which has its receipt stored in the legacy format. It'll
+// be removed after users have migrated their freezer databases.
+func readLegacyLogs(db ethdb.Reader, hash common.Hash, number uint64, config *params.ChainConfig) [][]*types.Log {
+	receipts := ReadReceipts(db, hash, number, config)
+	if receipts == nil {
+		return nil
+	}
+	logs := make([][]*types.Log, len(receipts))
+	for i, receipt := range receipts {
+		logs[i] = receipt.Logs
+	}
+	return logs
+}
+
 // ReadBlock retrieves an entire block corresponding to the hash, assembling it
 // back from the stored header and body. If either the header or body could not
 // be retrieved nil is returned.

core/rawdb/accessors_chain_test.go

@@ -744,7 +744,7 @@ func TestReadLogs(t *testing.T) {
 	// Insert the receipt slice into the database and check presence
 	WriteReceipts(db, hash, 0, receipts)
 
-	logs := ReadLogs(db, hash, 0)
+	logs := ReadLogs(db, hash, 0, params.TestChainConfig)
 	if len(logs) == 0 {
 		t.Fatalf("no logs returned")
 	}

core/rawdb/chain_iterator.go

@@ -44,24 +44,29 @@ func InitDatabaseFromFreezer(db ethdb.Database) {
 		logged = start.Add(-7 * time.Second) // Unindex during import is fast, don't double log
 		hash   common.Hash
 	)
-	for i := uint64(0); i < frozen; i++ {
-		// Since the freezer has all data in sequential order on a file,
-		// it would be 'neat' to read more data in one go, and let the
-		// freezerdb return N items (e.g up to 1000 items per go)
-		// That would require an API change in Ancients though
-		if h, err := db.Ancient(freezerHashTable, i); err != nil {
+	for i := uint64(0); i < frozen; {
+		// We read 100K hashes at a time, for a total of 3.2M
+		count := uint64(100_000)
+		if i+count > frozen {
+			count = frozen - i
+		}
+		data, err := db.AncientRange(freezerHashTable, i, count, 32*count)
+		if err != nil {
 			log.Crit("Failed to init database from freezer", "err", err)
-		} else {
+		}
+		for j, h := range data {
+			number := i + uint64(j)
 			hash = common.BytesToHash(h)
-		}
-		WriteHeaderNumber(batch, hash, i)
-		// If enough data was accumulated in memory or we're at the last block, dump to disk
-		if batch.ValueSize() > ethdb.IdealBatchSize {
-			if err := batch.Write(); err != nil {
-				log.Crit("Failed to write data to db", "err", err)
+			WriteHeaderNumber(batch, hash, number)
+			// If enough data was accumulated in memory or we're at the last block, dump to disk
+			if batch.ValueSize() > ethdb.IdealBatchSize {
+				if err := batch.Write(); err != nil {
+					log.Crit("Failed to write data to db", "err", err)
+				}
+				batch.Reset()
 			}
-			batch.Reset()
 		}
+		i += uint64(len(data))
 		// If we've spent too much time already, notify the user of what we're doing
 		if time.Since(logged) > 8*time.Second {
 			log.Info("Initializing database from freezer", "total", frozen, "number", i, "hash", hash, "elapsed", common.PrettyDuration(time.Since(start)))

core/rawdb/freezer_batch.go

@@ -118,7 +118,7 @@ func (batch *freezerTableBatch) reset() {
 // existing data.
 func (batch *freezerTableBatch) Append(item uint64, data interface{}) error {
 	if item != batch.curItem {
-		return errOutOrderInsertion
+		return fmt.Errorf("%w: have %d want %d", errOutOrderInsertion, item, batch.curItem)
 	}
 
 	// Encode the item.
@@ -138,7 +138,7 @@ func (batch *freezerTableBatch) Append(item uint64, data interface{}) error {
 // existing data.
 func (batch *freezerTableBatch) AppendRaw(item uint64, blob []byte) error {
 	if item != batch.curItem {
-		return errOutOrderInsertion
+		return fmt.Errorf("%w: have %d want %d", errOutOrderInsertion, item, batch.curItem)
 	}
 
 	encItem := blob

core/rawdb/freezer_test.go

@@ -246,7 +246,7 @@ func TestFreezerConcurrentModifyTruncate(t *testing.T) {
 		if truncateErr != nil {
 			t.Fatal("concurrent truncate failed:", err)
 		}
-		if !(modifyErr == nil || modifyErr == errOutOrderInsertion) {
+		if !(errors.Is(modifyErr, nil) || errors.Is(modifyErr, errOutOrderInsertion)) {
 			t.Fatal("wrong error from concurrent modify:", modifyErr)
 		}
 		checkAncientCount(t, f, "test", 10)