Add vendor dir (#16) (#4)

* Add vendor dir so builds dont require dep

* Pin specific version go-eth version

vendor/github.com/btcsuite/btcd/database/ffldb/README.md

@@ -0,0 +1,40 @@
+ffldb
+=====
+
+[![Build Status](https://travis-ci.org/btcsuite/btcd.png?branch=master)](https://travis-ci.org/btcsuite/btcd)
+[![ISC License](http://img.shields.io/badge/license-ISC-blue.svg)](http://copyfree.org)
+[![GoDoc](https://godoc.org/github.com/btcsuite/btcd/database/ffldb?status.png)](http://godoc.org/github.com/btcsuite/btcd/database/ffldb)
+
+Package ffldb implements a driver for the database package that uses leveldb for
+the backing metadata and flat files for block storage.
+
+This driver is the recommended driver for use with btcd.  It makes use leveldb
+for the metadata, flat files for block storage, and checksums in key areas to
+ensure data integrity.
+
+Package ffldb is licensed under the copyfree ISC license.
+
+## Usage
+
+This package is a driver to the database package and provides the database type
+of "ffldb".  The parameters the Open and Create functions take are the
+database path as a string and the block network.
+
+```Go
+db, err := database.Open("ffldb", "path/to/database", wire.MainNet)
+if err != nil {
+	// Handle error
+}
+```
+
+```Go
+db, err := database.Create("ffldb", "path/to/database", wire.MainNet)
+if err != nil {
+	// Handle error
+}
+```
+
+## License
+
+Package ffldb is licensed under the [copyfree](http://copyfree.org) ISC
+License.

vendor/github.com/btcsuite/btcd/database/ffldb/bench_test.go

@@ -0,0 +1,97 @@
+// Copyright (c) 2015-2016 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package ffldb
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/btcsuite/btcd/chaincfg"
+	"github.com/btcsuite/btcd/database"
+	"github.com/btcsuite/btcutil"
+)
+
+// BenchmarkBlockHeader benchmarks how long it takes to load the mainnet genesis
+// block header.
+func BenchmarkBlockHeader(b *testing.B) {
+	// Start by creating a new database and populating it with the mainnet
+	// genesis block.
+	dbPath := filepath.Join(os.TempDir(), "ffldb-benchblkhdr")
+	_ = os.RemoveAll(dbPath)
+	db, err := database.Create("ffldb", dbPath, blockDataNet)
+	if err != nil {
+		b.Fatal(err)
+	}
+	defer os.RemoveAll(dbPath)
+	defer db.Close()
+	err = db.Update(func(tx database.Tx) error {
+		block := btcutil.NewBlock(chaincfg.MainNetParams.GenesisBlock)
+		return tx.StoreBlock(block)
+	})
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	b.ReportAllocs()
+	b.ResetTimer()
+	err = db.View(func(tx database.Tx) error {
+		blockHash := chaincfg.MainNetParams.GenesisHash
+		for i := 0; i < b.N; i++ {
+			_, err := tx.FetchBlockHeader(blockHash)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	// Don't benchmark teardown.
+	b.StopTimer()
+}
+
+// BenchmarkBlockHeader benchmarks how long it takes to load the mainnet genesis
+// block.
+func BenchmarkBlock(b *testing.B) {
+	// Start by creating a new database and populating it with the mainnet
+	// genesis block.
+	dbPath := filepath.Join(os.TempDir(), "ffldb-benchblk")
+	_ = os.RemoveAll(dbPath)
+	db, err := database.Create("ffldb", dbPath, blockDataNet)
+	if err != nil {
+		b.Fatal(err)
+	}
+	defer os.RemoveAll(dbPath)
+	defer db.Close()
+	err = db.Update(func(tx database.Tx) error {
+		block := btcutil.NewBlock(chaincfg.MainNetParams.GenesisBlock)
+		return tx.StoreBlock(block)
+	})
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	b.ReportAllocs()
+	b.ResetTimer()
+	err = db.View(func(tx database.Tx) error {
+		blockHash := chaincfg.MainNetParams.GenesisHash
+		for i := 0; i < b.N; i++ {
+			_, err := tx.FetchBlock(blockHash)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	// Don't benchmark teardown.
+	b.StopTimer()
+}

vendor/github.com/btcsuite/btcd/database/ffldb/blockio.go

@@ -0,0 +1,769 @@
+// Copyright (c) 2015-2016 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+// This file contains the implementation functions for reading, writing, and
+// otherwise working with the flat files that house the actual blocks.
+
+package ffldb
+
+import (
+	"container/list"
+	"encoding/binary"
+	"fmt"
+	"hash/crc32"
+	"io"
+	"os"
+	"path/filepath"
+	"sync"
+
+	"github.com/btcsuite/btcd/chaincfg/chainhash"
+	"github.com/btcsuite/btcd/database"
+	"github.com/btcsuite/btcd/wire"
+)
+
+const (
+	// The Bitcoin protocol encodes block height as int32, so max number of
+	// blocks is 2^31.  Max block size per the protocol is 32MiB per block.
+	// So the theoretical max at the time this comment was written is 64PiB
+	// (pebibytes).  With files @ 512MiB each, this would require a maximum
+	// of 134,217,728 files.  Thus, choose 9 digits of precision for the
+	// filenames.  An additional benefit is 9 digits provides 10^9 files @
+	// 512MiB each for a total of ~476.84PiB (roughly 7.4 times the current
+	// theoretical max), so there is room for the max block size to grow in
+	// the future.
+	blockFilenameTemplate = "%09d.fdb"
+
+	// maxOpenFiles is the max number of open files to maintain in the
+	// open blocks cache.  Note that this does not include the current
+	// write file, so there will typically be one more than this value open.
+	maxOpenFiles = 25
+
+	// maxBlockFileSize is the maximum size for each file used to store
+	// blocks.
+	//
+	// NOTE: The current code uses uint32 for all offsets, so this value
+	// must be less than 2^32 (4 GiB).  This is also why it's a typed
+	// constant.
+	maxBlockFileSize uint32 = 512 * 1024 * 1024 // 512 MiB
+
+	// blockLocSize is the number of bytes the serialized block location
+	// data that is stored in the block index.
+	//
+	// The serialized block location format is:
+	//
+	//  [0:4]  Block file (4 bytes)
+	//  [4:8]  File offset (4 bytes)
+	//  [8:12] Block length (4 bytes)
+	blockLocSize = 12
+)
+
+var (
+	// castagnoli houses the Catagnoli polynomial used for CRC-32 checksums.
+	castagnoli = crc32.MakeTable(crc32.Castagnoli)
+)
+
+// filer is an interface which acts very similar to a *os.File and is typically
+// implemented by it.  It exists so the test code can provide mock files for
+// properly testing corruption and file system issues.
+type filer interface {
+	io.Closer
+	io.WriterAt
+	io.ReaderAt
+	Truncate(size int64) error
+	Sync() error
+}
+
+// lockableFile represents a block file on disk that has been opened for either
+// read or read/write access.  It also contains a read-write mutex to support
+// multiple concurrent readers.
+type lockableFile struct {
+	sync.RWMutex
+	file filer
+}
+
+// writeCursor represents the current file and offset of the block file on disk
+// for performing all writes. It also contains a read-write mutex to support
+// multiple concurrent readers which can reuse the file handle.
+type writeCursor struct {
+	sync.RWMutex
+
+	// curFile is the current block file that will be appended to when
+	// writing new blocks.
+	curFile *lockableFile
+
+	// curFileNum is the current block file number and is used to allow
+	// readers to use the same open file handle.
+	curFileNum uint32
+
+	// curOffset is the offset in the current write block file where the
+	// next new block will be written.
+	curOffset uint32
+}
+
+// blockStore houses information used to handle reading and writing blocks (and
+// part of blocks) into flat files with support for multiple concurrent readers.
+type blockStore struct {
+	// network is the specific network to use in the flat files for each
+	// block.
+	network wire.BitcoinNet
+
+	// basePath is the base path used for the flat block files and metadata.
+	basePath string
+
+	// maxBlockFileSize is the maximum size for each file used to store
+	// blocks.  It is defined on the store so the whitebox tests can
+	// override the value.
+	maxBlockFileSize uint32
+
+	// The following fields are related to the flat files which hold the
+	// actual blocks.   The number of open files is limited by maxOpenFiles.
+	//
+	// obfMutex protects concurrent access to the openBlockFiles map.  It is
+	// a RWMutex so multiple readers can simultaneously access open files.
+	//
+	// openBlockFiles houses the open file handles for existing block files
+	// which have been opened read-only along with an individual RWMutex.
+	// This scheme allows multiple concurrent readers to the same file while
+	// preventing the file from being closed out from under them.
+	//
+	// lruMutex protects concurrent access to the least recently used list
+	// and lookup map.
+	//
+	// openBlocksLRU tracks how the open files are refenced by pushing the
+	// most recently used files to the front of the list thereby trickling
+	// the least recently used files to end of the list.  When a file needs
+	// to be closed due to exceeding the the max number of allowed open
+	// files, the one at the end of the list is closed.
+	//
+	// fileNumToLRUElem is a mapping between a specific block file number
+	// and the associated list element on the least recently used list.
+	//
+	// Thus, with the combination of these fields, the database supports
+	// concurrent non-blocking reads across multiple and individual files
+	// along with intelligently limiting the number of open file handles by
+	// closing the least recently used files as needed.
+	//
+	// NOTE: The locking order used throughout is well-defined and MUST be
+	// followed.  Failure to do so could lead to deadlocks.  In particular,
+	// the locking order is as follows:
+	//   1) obfMutex
+	//   2) lruMutex
+	//   3) writeCursor mutex
+	//   4) specific file mutexes
+	//
+	// None of the mutexes are required to be locked at the same time, and
+	// often aren't.  However, if they are to be locked simultaneously, they
+	// MUST be locked in the order previously specified.
+	//
+	// Due to the high performance and multi-read concurrency requirements,
+	// write locks should only be held for the minimum time necessary.
+	obfMutex         sync.RWMutex
+	lruMutex         sync.Mutex
+	openBlocksLRU    *list.List // Contains uint32 block file numbers.
+	fileNumToLRUElem map[uint32]*list.Element
+	openBlockFiles   map[uint32]*lockableFile
+
+	// writeCursor houses the state for the current file and location that
+	// new blocks are written to.
+	writeCursor *writeCursor
+
+	// These functions are set to openFile, openWriteFile, and deleteFile by
+	// default, but are exposed here to allow the whitebox tests to replace
+	// them when working with mock files.
+	openFileFunc      func(fileNum uint32) (*lockableFile, error)
+	openWriteFileFunc func(fileNum uint32) (filer, error)
+	deleteFileFunc    func(fileNum uint32) error
+}
+
+// blockLocation identifies a particular block file and location.
+type blockLocation struct {
+	blockFileNum uint32
+	fileOffset   uint32
+	blockLen     uint32
+}
+
+// deserializeBlockLoc deserializes the passed serialized block location
+// information.  This is data stored into the block index metadata for each
+// block.  The serialized data passed to this function MUST be at least
+// blockLocSize bytes or it will panic.  The error check is avoided here because
+// this information will always be coming from the block index which includes a
+// checksum to detect corruption.  Thus it is safe to use this unchecked here.
+func deserializeBlockLoc(serializedLoc []byte) blockLocation {
+	// The serialized block location format is:
+	//
+	//  [0:4]  Block file (4 bytes)
+	//  [4:8]  File offset (4 bytes)
+	//  [8:12] Block length (4 bytes)
+	return blockLocation{
+		blockFileNum: byteOrder.Uint32(serializedLoc[0:4]),
+		fileOffset:   byteOrder.Uint32(serializedLoc[4:8]),
+		blockLen:     byteOrder.Uint32(serializedLoc[8:12]),
+	}
+}
+
+// serializeBlockLoc returns the serialization of the passed block location.
+// This is data to be stored into the block index metadata for each block.
+func serializeBlockLoc(loc blockLocation) []byte {
+	// The serialized block location format is:
+	//
+	//  [0:4]  Block file (4 bytes)
+	//  [4:8]  File offset (4 bytes)
+	//  [8:12] Block length (4 bytes)
+	var serializedData [12]byte
+	byteOrder.PutUint32(serializedData[0:4], loc.blockFileNum)
+	byteOrder.PutUint32(serializedData[4:8], loc.fileOffset)
+	byteOrder.PutUint32(serializedData[8:12], loc.blockLen)
+	return serializedData[:]
+}
+
+// blockFilePath return the file path for the provided block file number.
+func blockFilePath(dbPath string, fileNum uint32) string {
+	fileName := fmt.Sprintf(blockFilenameTemplate, fileNum)
+	return filepath.Join(dbPath, fileName)
+}
+
+// openWriteFile returns a file handle for the passed flat file number in
+// read/write mode.  The file will be created if needed.  It is typically used
+// for the current file that will have all new data appended.  Unlike openFile,
+// this function does not keep track of the open file and it is not subject to
+// the maxOpenFiles limit.
+func (s *blockStore) openWriteFile(fileNum uint32) (filer, error) {
+	// The current block file needs to be read-write so it is possible to
+	// append to it.  Also, it shouldn't be part of the least recently used
+	// file.
+	filePath := blockFilePath(s.basePath, fileNum)
+	file, err := os.OpenFile(filePath, os.O_RDWR|os.O_CREATE, 0666)
+	if err != nil {
+		str := fmt.Sprintf("failed to open file %q: %v", filePath, err)
+		return nil, makeDbErr(database.ErrDriverSpecific, str, err)
+	}
+
+	return file, nil
+}
+
+// openFile returns a read-only file handle for the passed flat file number.
+// The function also keeps track of the open files, performs least recently
+// used tracking, and limits the number of open files to maxOpenFiles by closing
+// the least recently used file as needed.
+//
+// This function MUST be called with the overall files mutex (s.obfMutex) locked
+// for WRITES.
+func (s *blockStore) openFile(fileNum uint32) (*lockableFile, error) {
+	// Open the appropriate file as read-only.
+	filePath := blockFilePath(s.basePath, fileNum)
+	file, err := os.Open(filePath)
+	if err != nil {
+		return nil, makeDbErr(database.ErrDriverSpecific, err.Error(),
+			err)
+	}
+	blockFile := &lockableFile{file: file}
+
+	// Close the least recently used file if the file exceeds the max
+	// allowed open files.  This is not done until after the file open in
+	// case the file fails to open, there is no need to close any files.
+	//
+	// A write lock is required on the LRU list here to protect against
+	// modifications happening as already open files are read from and
+	// shuffled to the front of the list.
+	//
+	// Also, add the file that was just opened to the front of the least
+	// recently used list to indicate it is the most recently used file and
+	// therefore should be closed last.
+	s.lruMutex.Lock()
+	lruList := s.openBlocksLRU
+	if lruList.Len() >= maxOpenFiles {
+		lruFileNum := lruList.Remove(lruList.Back()).(uint32)
+		oldBlockFile := s.openBlockFiles[lruFileNum]
+
+		// Close the old file under the write lock for the file in case
+		// any readers are currently reading from it so it's not closed
+		// out from under them.
+		oldBlockFile.Lock()
+		_ = oldBlockFile.file.Close()
+		oldBlockFile.Unlock()
+
+		delete(s.openBlockFiles, lruFileNum)
+		delete(s.fileNumToLRUElem, lruFileNum)
+	}
+	s.fileNumToLRUElem[fileNum] = lruList.PushFront(fileNum)
+	s.lruMutex.Unlock()
+
+	// Store a reference to it in the open block files map.
+	s.openBlockFiles[fileNum] = blockFile
+
+	return blockFile, nil
+}
+
+// deleteFile removes the block file for the passed flat file number.  The file
+// must already be closed and it is the responsibility of the caller to do any
+// other state cleanup necessary.
+func (s *blockStore) deleteFile(fileNum uint32) error {
+	filePath := blockFilePath(s.basePath, fileNum)
+	if err := os.Remove(filePath); err != nil {
+		return makeDbErr(database.ErrDriverSpecific, err.Error(), err)
+	}
+
+	return nil
+}
+
+// blockFile attempts to return an existing file handle for the passed flat file
+// number if it is already open as well as marking it as most recently used.  It
+// will also open the file when it's not already open subject to the rules
+// described in openFile.
+//
+// NOTE: The returned block file will already have the read lock acquired and
+// the caller MUST call .RUnlock() to release it once it has finished all read
+// operations.  This is necessary because otherwise it would be possible for a
+// separate goroutine to close the file after it is returned from here, but
+// before the caller has acquired a read lock.
+func (s *blockStore) blockFile(fileNum uint32) (*lockableFile, error) {
+	// When the requested block file is open for writes, return it.
+	wc := s.writeCursor
+	wc.RLock()
+	if fileNum == wc.curFileNum && wc.curFile.file != nil {
+		obf := wc.curFile
+		obf.RLock()
+		wc.RUnlock()
+		return obf, nil
+	}
+	wc.RUnlock()
+
+	// Try to return an open file under the overall files read lock.
+	s.obfMutex.RLock()
+	if obf, ok := s.openBlockFiles[fileNum]; ok {
+		s.lruMutex.Lock()
+		s.openBlocksLRU.MoveToFront(s.fileNumToLRUElem[fileNum])
+		s.lruMutex.Unlock()
+
+		obf.RLock()
+		s.obfMutex.RUnlock()
+		return obf, nil
+	}
+	s.obfMutex.RUnlock()
+
+	// Since the file isn't open already, need to check the open block files
+	// map again under write lock in case multiple readers got here and a
+	// separate one is already opening the file.
+	s.obfMutex.Lock()
+	if obf, ok := s.openBlockFiles[fileNum]; ok {
+		obf.RLock()
+		s.obfMutex.Unlock()
+		return obf, nil
+	}
+
+	// The file isn't open, so open it while potentially closing the least
+	// recently used one as needed.
+	obf, err := s.openFileFunc(fileNum)
+	if err != nil {
+		s.obfMutex.Unlock()
+		return nil, err
+	}
+	obf.RLock()
+	s.obfMutex.Unlock()
+	return obf, nil
+}
+
+// writeData is a helper function for writeBlock which writes the provided data
+// at the current write offset and updates the write cursor accordingly.  The
+// field name parameter is only used when there is an error to provide a nicer
+// error message.
+//
+// The write cursor will be advanced the number of bytes actually written in the
+// event of failure.
+//
+// NOTE: This function MUST be called with the write cursor current file lock
+// held and must only be called during a write transaction so it is effectively
+// locked for writes.  Also, the write cursor current file must NOT be nil.
+func (s *blockStore) writeData(data []byte, fieldName string) error {
+	wc := s.writeCursor
+	n, err := wc.curFile.file.WriteAt(data, int64(wc.curOffset))
+	wc.curOffset += uint32(n)
+	if err != nil {
+		str := fmt.Sprintf("failed to write %s to file %d at "+
+			"offset %d: %v", fieldName, wc.curFileNum,
+			wc.curOffset-uint32(n), err)
+		return makeDbErr(database.ErrDriverSpecific, str, err)
+	}
+
+	return nil
+}
+
+// writeBlock appends the specified raw block bytes to the store's write cursor
+// location and increments it accordingly.  When the block would exceed the max
+// file size for the current flat file, this function will close the current
+// file, create the next file, update the write cursor, and write the block to
+// the new file.
+//
+// The write cursor will also be advanced the number of bytes actually written
+// in the event of failure.
+//
+// Format: <network><block length><serialized block><checksum>
+func (s *blockStore) writeBlock(rawBlock []byte) (blockLocation, error) {
+	// Compute how many bytes will be written.
+	// 4 bytes each for block network + 4 bytes for block length +
+	// length of raw block + 4 bytes for checksum.
+	blockLen := uint32(len(rawBlock))
+	fullLen := blockLen + 12
+
+	// Move to the next block file if adding the new block would exceed the
+	// max allowed size for the current block file.  Also detect overflow
+	// to be paranoid, even though it isn't possible currently, numbers
+	// might change in the future to make it possible.
+	//
+	// NOTE: The writeCursor.offset field isn't protected by the mutex
+	// since it's only read/changed during this function which can only be
+	// called during a write transaction, of which there can be only one at
+	// a time.
+	wc := s.writeCursor
+	finalOffset := wc.curOffset + fullLen
+	if finalOffset < wc.curOffset || finalOffset > s.maxBlockFileSize {
+		// This is done under the write cursor lock since the curFileNum
+		// field is accessed elsewhere by readers.
+		//
+		// Close the current write file to force a read-only reopen
+		// with LRU tracking.  The close is done under the write lock
+		// for the file to prevent it from being closed out from under
+		// any readers currently reading from it.
+		wc.Lock()
+		wc.curFile.Lock()
+		if wc.curFile.file != nil {
+			_ = wc.curFile.file.Close()
+			wc.curFile.file = nil
+		}
+		wc.curFile.Unlock()
+
+		// Start writes into next file.
+		wc.curFileNum++
+		wc.curOffset = 0
+		wc.Unlock()
+	}
+
+	// All writes are done under the write lock for the file to ensure any
+	// readers are finished and blocked first.
+	wc.curFile.Lock()
+	defer wc.curFile.Unlock()
+
+	// Open the current file if needed.  This will typically only be the
+	// case when moving to the next file to write to or on initial database
+	// load.  However, it might also be the case if rollbacks happened after
+	// file writes started during a transaction commit.
+	if wc.curFile.file == nil {
+		file, err := s.openWriteFileFunc(wc.curFileNum)
+		if err != nil {
+			return blockLocation{}, err
+		}
+		wc.curFile.file = file
+	}
+
+	// Bitcoin network.
+	origOffset := wc.curOffset
+	hasher := crc32.New(castagnoli)
+	var scratch [4]byte
+	byteOrder.PutUint32(scratch[:], uint32(s.network))
+	if err := s.writeData(scratch[:], "network"); err != nil {
+		return blockLocation{}, err
+	}
+	_, _ = hasher.Write(scratch[:])
+
+	// Block length.
+	byteOrder.PutUint32(scratch[:], blockLen)
+	if err := s.writeData(scratch[:], "block length"); err != nil {
+		return blockLocation{}, err
+	}
+	_, _ = hasher.Write(scratch[:])
+
+	// Serialized block.
+	if err := s.writeData(rawBlock[:], "block"); err != nil {
+		return blockLocation{}, err
+	}
+	_, _ = hasher.Write(rawBlock)
+
+	// Castagnoli CRC-32 as a checksum of all the previous.
+	if err := s.writeData(hasher.Sum(nil), "checksum"); err != nil {
+		return blockLocation{}, err
+	}
+
+	loc := blockLocation{
+		blockFileNum: wc.curFileNum,
+		fileOffset:   origOffset,
+		blockLen:     fullLen,
+	}
+	return loc, nil
+}
+
+// readBlock reads the specified block record and returns the serialized block.
+// It ensures the integrity of the block data by checking that the serialized
+// network matches the current network associated with the block store and
+// comparing the calculated checksum against the one stored in the flat file.
+// This function also automatically handles all file management such as opening
+// and closing files as necessary to stay within the maximum allowed open files
+// limit.
+//
+// Returns ErrDriverSpecific if the data fails to read for any reason and
+// ErrCorruption if the checksum of the read data doesn't match the checksum
+// read from the file.
+//
+// Format: <network><block length><serialized block><checksum>
+func (s *blockStore) readBlock(hash *chainhash.Hash, loc blockLocation) ([]byte, error) {
+	// Get the referenced block file handle opening the file as needed.  The
+	// function also handles closing files as needed to avoid going over the
+	// max allowed open files.
+	blockFile, err := s.blockFile(loc.blockFileNum)
+	if err != nil {
+		return nil, err
+	}
+
+	serializedData := make([]byte, loc.blockLen)
+	n, err := blockFile.file.ReadAt(serializedData, int64(loc.fileOffset))
+	blockFile.RUnlock()
+	if err != nil {
+		str := fmt.Sprintf("failed to read block %s from file %d, "+
+			"offset %d: %v", hash, loc.blockFileNum, loc.fileOffset,
+			err)
+		return nil, makeDbErr(database.ErrDriverSpecific, str, err)
+	}
+
+	// Calculate the checksum of the read data and ensure it matches the
+	// serialized checksum.  This will detect any data corruption in the
+	// flat file without having to do much more expensive merkle root
+	// calculations on the loaded block.
+	serializedChecksum := binary.BigEndian.Uint32(serializedData[n-4:])
+	calculatedChecksum := crc32.Checksum(serializedData[:n-4], castagnoli)
+	if serializedChecksum != calculatedChecksum {
+		str := fmt.Sprintf("block data for block %s checksum "+
+			"does not match - got %x, want %x", hash,
+			calculatedChecksum, serializedChecksum)
+		return nil, makeDbErr(database.ErrCorruption, str, nil)
+	}
+
+	// The network associated with the block must match the current active
+	// network, otherwise somebody probably put the block files for the
+	// wrong network in the directory.
+	serializedNet := byteOrder.Uint32(serializedData[:4])
+	if serializedNet != uint32(s.network) {
+		str := fmt.Sprintf("block data for block %s is for the "+
+			"wrong network - got %d, want %d", hash, serializedNet,
+			uint32(s.network))
+		return nil, makeDbErr(database.ErrDriverSpecific, str, nil)
+	}
+
+	// The raw block excludes the network, length of the block, and
+	// checksum.
+	return serializedData[8 : n-4], nil
+}
+
+// readBlockRegion reads the specified amount of data at the provided offset for
+// a given block location.  The offset is relative to the start of the
+// serialized block (as opposed to the beginning of the block record).  This
+// function automatically handles all file management such as opening and
+// closing files as necessary to stay within the maximum allowed open files
+// limit.
+//
+// Returns ErrDriverSpecific if the data fails to read for any reason.
+func (s *blockStore) readBlockRegion(loc blockLocation, offset, numBytes uint32) ([]byte, error) {
+	// Get the referenced block file handle opening the file as needed.  The
+	// function also handles closing files as needed to avoid going over the
+	// max allowed open files.
+	blockFile, err := s.blockFile(loc.blockFileNum)
+	if err != nil {
+		return nil, err
+	}
+
+	// Regions are offsets into the actual block, however the serialized
+	// data for a block includes an initial 4 bytes for network + 4 bytes
+	// for block length.  Thus, add 8 bytes to adjust.
+	readOffset := loc.fileOffset + 8 + offset
+	serializedData := make([]byte, numBytes)
+	_, err = blockFile.file.ReadAt(serializedData, int64(readOffset))
+	blockFile.RUnlock()
+	if err != nil {
+		str := fmt.Sprintf("failed to read region from block file %d, "+
+			"offset %d, len %d: %v", loc.blockFileNum, readOffset,
+			numBytes, err)
+		return nil, makeDbErr(database.ErrDriverSpecific, str, err)
+	}
+
+	return serializedData, nil
+}
+
+// syncBlocks performs a file system sync on the flat file associated with the
+// store's current write cursor.  It is safe to call even when there is not a
+// current write file in which case it will have no effect.
+//
+// This is used when flushing cached metadata updates to disk to ensure all the
+// block data is fully written before updating the metadata.  This ensures the
+// metadata and block data can be properly reconciled in failure scenarios.
+func (s *blockStore) syncBlocks() error {
+	wc := s.writeCursor
+	wc.RLock()
+	defer wc.RUnlock()
+
+	// Nothing to do if there is no current file associated with the write
+	// cursor.
+	wc.curFile.RLock()
+	defer wc.curFile.RUnlock()
+	if wc.curFile.file == nil {
+		return nil
+	}
+
+	// Sync the file to disk.
+	if err := wc.curFile.file.Sync(); err != nil {
+		str := fmt.Sprintf("failed to sync file %d: %v", wc.curFileNum,
+			err)
+		return makeDbErr(database.ErrDriverSpecific, str, err)
+	}
+
+	return nil
+}
+
+// handleRollback rolls the block files on disk back to the provided file number
+// and offset.  This involves potentially deleting and truncating the files that
+// were partially written.
+//
+// There are effectively two scenarios to consider here:
+//   1) Transient write failures from which recovery is possible
+//   2) More permanent failures such as hard disk death and/or removal
+//
+// In either case, the write cursor will be repositioned to the old block file
+// offset regardless of any other errors that occur while attempting to undo
+// writes.
+//
+// For the first scenario, this will lead to any data which failed to be undone
+// being overwritten and thus behaves as desired as the system continues to run.
+//
+// For the second scenario, the metadata which stores the current write cursor
+// position within the block files will not have been updated yet and thus if
+// the system eventually recovers (perhaps the hard drive is reconnected), it
+// will also lead to any data which failed to be undone being overwritten and
+// thus behaves as desired.
+//
+// Therefore, any errors are simply logged at a warning level rather than being
+// returned since there is nothing more that could be done about it anyways.
+func (s *blockStore) handleRollback(oldBlockFileNum, oldBlockOffset uint32) {
+	// Grab the write cursor mutex since it is modified throughout this
+	// function.
+	wc := s.writeCursor
+	wc.Lock()
+	defer wc.Unlock()
+
+	// Nothing to do if the rollback point is the same as the current write
+	// cursor.
+	if wc.curFileNum == oldBlockFileNum && wc.curOffset == oldBlockOffset {
+		return
+	}
+
+	// Regardless of any failures that happen below, reposition the write
+	// cursor to the old block file and offset.
+	defer func() {
+		wc.curFileNum = oldBlockFileNum
+		wc.curOffset = oldBlockOffset
+	}()
+
+	log.Debugf("ROLLBACK: Rolling back to file %d, offset %d",
+		oldBlockFileNum, oldBlockOffset)
+
+	// Close the current write file if it needs to be deleted.  Then delete
+	// all files that are newer than the provided rollback file while
+	// also moving the write cursor file backwards accordingly.
+	if wc.curFileNum > oldBlockFileNum {
+		wc.curFile.Lock()
+		if wc.curFile.file != nil {
+			_ = wc.curFile.file.Close()
+			wc.curFile.file = nil
+		}
+		wc.curFile.Unlock()
+	}
+	for ; wc.curFileNum > oldBlockFileNum; wc.curFileNum-- {
+		if err := s.deleteFileFunc(wc.curFileNum); err != nil {
+			log.Warnf("ROLLBACK: Failed to delete block file "+
+				"number %d: %v", wc.curFileNum, err)
+			return
+		}
+	}
+
+	// Open the file for the current write cursor if needed.
+	wc.curFile.Lock()
+	if wc.curFile.file == nil {
+		obf, err := s.openWriteFileFunc(wc.curFileNum)
+		if err != nil {
+			wc.curFile.Unlock()
+			log.Warnf("ROLLBACK: %v", err)
+			return
+		}
+		wc.curFile.file = obf
+	}
+
+	// Truncate the to the provided rollback offset.
+	if err := wc.curFile.file.Truncate(int64(oldBlockOffset)); err != nil {
+		wc.curFile.Unlock()
+		log.Warnf("ROLLBACK: Failed to truncate file %d: %v",
+			wc.curFileNum, err)
+		return
+	}
+
+	// Sync the file to disk.
+	err := wc.curFile.file.Sync()
+	wc.curFile.Unlock()
+	if err != nil {
+		log.Warnf("ROLLBACK: Failed to sync file %d: %v",
+			wc.curFileNum, err)
+		return
+	}
+}
+
+// scanBlockFiles searches the database directory for all flat block files to
+// find the end of the most recent file.  This position is considered the
+// current write cursor which is also stored in the metadata.  Thus, it is used
+// to detect unexpected shutdowns in the middle of writes so the block files
+// can be reconciled.
+func scanBlockFiles(dbPath string) (int, uint32) {
+	lastFile := -1
+	fileLen := uint32(0)
+	for i := 0; ; i++ {
+		filePath := blockFilePath(dbPath, uint32(i))
+		st, err := os.Stat(filePath)
+		if err != nil {
+			break
+		}
+		lastFile = i
+
+		fileLen = uint32(st.Size())
+	}
+
+	log.Tracef("Scan found latest block file #%d with length %d", lastFile,
+		fileLen)
+	return lastFile, fileLen
+}
+
+// newBlockStore returns a new block store with the current block file number
+// and offset set and all fields initialized.
+func newBlockStore(basePath string, network wire.BitcoinNet) *blockStore {
+	// Look for the end of the latest block to file to determine what the
+	// write cursor position is from the viewpoing of the block files on
+	// disk.
+	fileNum, fileOff := scanBlockFiles(basePath)
+	if fileNum == -1 {
+		fileNum = 0
+		fileOff = 0
+	}
+
+	store := &blockStore{
+		network:          network,
+		basePath:         basePath,
+		maxBlockFileSize: maxBlockFileSize,
+		openBlockFiles:   make(map[uint32]*lockableFile),
+		openBlocksLRU:    list.New(),
+		fileNumToLRUElem: make(map[uint32]*list.Element),
+
+		writeCursor: &writeCursor{
+			curFile:    &lockableFile{},
+			curFileNum: uint32(fileNum),
+			curOffset:  fileOff,
+		},
+	}
+	store.openFileFunc = store.openFile
+	store.openWriteFileFunc = store.openWriteFile
+	store.deleteFileFunc = store.deleteFile
+	return store
+}

vendor/github.com/btcsuite/btcd/database/ffldb/dbcache.go

@@ -0,0 +1,660 @@
+// Copyright (c) 2015-2016 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package ffldb
+
+import (
+	"bytes"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/btcsuite/btcd/database/internal/treap"
+	"github.com/btcsuite/goleveldb/leveldb"
+	"github.com/btcsuite/goleveldb/leveldb/iterator"
+	"github.com/btcsuite/goleveldb/leveldb/util"
+)
+
+const (
+	// defaultCacheSize is the default size for the database cache.
+	defaultCacheSize = 100 * 1024 * 1024 // 100 MB
+
+	// defaultFlushSecs is the default number of seconds to use as a
+	// threshold in between database cache flushes when the cache size has
+	// not been exceeded.
+	defaultFlushSecs = 300 // 5 minutes
+
+	// ldbBatchHeaderSize is the size of a leveldb batch header which
+	// includes the sequence header and record counter.
+	//
+	// ldbRecordIKeySize is the size of the ikey used internally by leveldb
+	// when appending a record to a batch.
+	//
+	// These are used to help preallocate space needed for a batch in one
+	// allocation instead of letting leveldb itself constantly grow it.
+	// This results in far less pressure on the GC and consequently helps
+	// prevent the GC from allocating a lot of extra unneeded space.
+	ldbBatchHeaderSize = 12
+	ldbRecordIKeySize  = 8
+)
+
+// ldbCacheIter wraps a treap iterator to provide the additional functionality
+// needed to satisfy the leveldb iterator.Iterator interface.
+type ldbCacheIter struct {
+	*treap.Iterator
+}
+
+// Enforce ldbCacheIterator implements the leveldb iterator.Iterator interface.
+var _ iterator.Iterator = (*ldbCacheIter)(nil)
+
+// Error is only provided to satisfy the iterator interface as there are no
+// errors for this memory-only structure.
+//
+// This is part of the leveldb iterator.Iterator interface implementation.
+func (iter *ldbCacheIter) Error() error {
+	return nil
+}
+
+// SetReleaser is only provided to satisfy the iterator interface as there is no
+// need to override it.
+//
+// This is part of the leveldb iterator.Iterator interface implementation.
+func (iter *ldbCacheIter) SetReleaser(releaser util.Releaser) {
+}
+
+// Release is only provided to satisfy the iterator interface.
+//
+// This is part of the leveldb iterator.Iterator interface implementation.
+func (iter *ldbCacheIter) Release() {
+}
+
+// newLdbCacheIter creates a new treap iterator for the given slice against the
+// pending keys for the passed cache snapshot and returns it wrapped in an
+// ldbCacheIter so it can be used as a leveldb iterator.
+func newLdbCacheIter(snap *dbCacheSnapshot, slice *util.Range) *ldbCacheIter {
+	iter := snap.pendingKeys.Iterator(slice.Start, slice.Limit)
+	return &ldbCacheIter{Iterator: iter}
+}
+
+// dbCacheIterator defines an iterator over the key/value pairs in the database
+// cache and underlying database.
+type dbCacheIterator struct {
+	cacheSnapshot *dbCacheSnapshot
+	dbIter        iterator.Iterator
+	cacheIter     iterator.Iterator
+	currentIter   iterator.Iterator
+	released      bool
+}
+
+// Enforce dbCacheIterator implements the leveldb iterator.Iterator interface.
+var _ iterator.Iterator = (*dbCacheIterator)(nil)
+
+// skipPendingUpdates skips any keys at the current database iterator position
+// that are being updated by the cache.  The forwards flag indicates the
+// direction the iterator is moving.
+func (iter *dbCacheIterator) skipPendingUpdates(forwards bool) {
+	for iter.dbIter.Valid() {
+		var skip bool
+		key := iter.dbIter.Key()
+		if iter.cacheSnapshot.pendingRemove.Has(key) {
+			skip = true
+		} else if iter.cacheSnapshot.pendingKeys.Has(key) {
+			skip = true
+		}
+		if !skip {
+			break
+		}
+
+		if forwards {
+			iter.dbIter.Next()
+		} else {
+			iter.dbIter.Prev()
+		}
+	}
+}
+
+// chooseIterator first skips any entries in the database iterator that are
+// being updated by the cache and sets the current iterator to the appropriate
+// iterator depending on their validity and the order they compare in while taking
+// into account the direction flag.  When the iterator is being moved forwards
+// and both iterators are valid, the iterator with the smaller key is chosen and
+// vice versa when the iterator is being moved backwards.
+func (iter *dbCacheIterator) chooseIterator(forwards bool) bool {
+	// Skip any keys at the current database iterator position that are
+	// being updated by the cache.
+	iter.skipPendingUpdates(forwards)
+
+	// When both iterators are exhausted, the iterator is exhausted too.
+	if !iter.dbIter.Valid() && !iter.cacheIter.Valid() {
+		iter.currentIter = nil
+		return false
+	}
+
+	// Choose the database iterator when the cache iterator is exhausted.
+	if !iter.cacheIter.Valid() {
+		iter.currentIter = iter.dbIter
+		return true
+	}
+
+	// Choose the cache iterator when the database iterator is exhausted.
+	if !iter.dbIter.Valid() {
+		iter.currentIter = iter.cacheIter
+		return true
+	}
+
+	// Both iterators are valid, so choose the iterator with either the
+	// smaller or larger key depending on the forwards flag.
+	compare := bytes.Compare(iter.dbIter.Key(), iter.cacheIter.Key())
+	if (forwards && compare > 0) || (!forwards && compare < 0) {
+		iter.currentIter = iter.cacheIter
+	} else {
+		iter.currentIter = iter.dbIter
+	}
+	return true
+}
+
+// First positions the iterator at the first key/value pair and returns whether
+// or not the pair exists.
+//
+// This is part of the leveldb iterator.Iterator interface implementation.
+func (iter *dbCacheIterator) First() bool {
+	// Seek to the first key in both the database and cache iterators and
+	// choose the iterator that is both valid and has the smaller key.
+	iter.dbIter.First()
+	iter.cacheIter.First()
+	return iter.chooseIterator(true)
+}
+
+// Last positions the iterator at the last key/value pair and returns whether or
+// not the pair exists.
+//
+// This is part of the leveldb iterator.Iterator interface implementation.
+func (iter *dbCacheIterator) Last() bool {
+	// Seek to the last key in both the database and cache iterators and
+	// choose the iterator that is both valid and has the larger key.
+	iter.dbIter.Last()
+	iter.cacheIter.Last()
+	return iter.chooseIterator(false)
+}
+
+// Next moves the iterator one key/value pair forward and returns whether or not
+// the pair exists.
+//
+// This is part of the leveldb iterator.Iterator interface implementation.
+func (iter *dbCacheIterator) Next() bool {
+	// Nothing to return if cursor is exhausted.
+	if iter.currentIter == nil {
+		return false
+	}
+
+	// Move the current iterator to the next entry and choose the iterator
+	// that is both valid and has the smaller key.
+	iter.currentIter.Next()
+	return iter.chooseIterator(true)
+}
+
+// Prev moves the iterator one key/value pair backward and returns whether or
+// not the pair exists.
+//
+// This is part of the leveldb iterator.Iterator interface implementation.
+func (iter *dbCacheIterator) Prev() bool {
+	// Nothing to return if cursor is exhausted.
+	if iter.currentIter == nil {
+		return false
+	}
+
+	// Move the current iterator to the previous entry and choose the
+	// iterator that is both valid and has the larger key.
+	iter.currentIter.Prev()
+	return iter.chooseIterator(false)
+}
+
+// Seek positions the iterator at the first key/value pair that is greater than
+// or equal to the passed seek key.  Returns false if no suitable key was found.
+//
+// This is part of the leveldb iterator.Iterator interface implementation.
+func (iter *dbCacheIterator) Seek(key []byte) bool {
+	// Seek to the provided key in both the database and cache iterators
+	// then choose the iterator that is both valid and has the larger key.
+	iter.dbIter.Seek(key)
+	iter.cacheIter.Seek(key)
+	return iter.chooseIterator(true)
+}
+
+// Valid indicates whether the iterator is positioned at a valid key/value pair.
+// It will be considered invalid when the iterator is newly created or exhausted.
+//
+// This is part of the leveldb iterator.Iterator interface implementation.
+func (iter *dbCacheIterator) Valid() bool {
+	return iter.currentIter != nil
+}
+
+// Key returns the current key the iterator is pointing to.
+//
+// This is part of the leveldb iterator.Iterator interface implementation.
+func (iter *dbCacheIterator) Key() []byte {
+	// Nothing to return if iterator is exhausted.
+	if iter.currentIter == nil {
+		return nil
+	}
+
+	return iter.currentIter.Key()
+}
+
+// Value returns the current value the iterator is pointing to.
+//
+// This is part of the leveldb iterator.Iterator interface implementation.
+func (iter *dbCacheIterator) Value() []byte {
+	// Nothing to return if iterator is exhausted.
+	if iter.currentIter == nil {
+		return nil
+	}
+
+	return iter.currentIter.Value()
+}
+
+// SetReleaser is only provided to satisfy the iterator interface as there is no
+// need to override it.
+//
+// This is part of the leveldb iterator.Iterator interface implementation.
+func (iter *dbCacheIterator) SetReleaser(releaser util.Releaser) {
+}
+
+// Release releases the iterator by removing the underlying treap iterator from
+// the list of active iterators against the pending keys treap.
+//
+// This is part of the leveldb iterator.Iterator interface implementation.
+func (iter *dbCacheIterator) Release() {
+	if !iter.released {
+		iter.dbIter.Release()
+		iter.cacheIter.Release()
+		iter.currentIter = nil
+		iter.released = true
+	}
+}
+
+// Error is only provided to satisfy the iterator interface as there are no
+// errors for this memory-only structure.
+//
+// This is part of the leveldb iterator.Iterator interface implementation.
+func (iter *dbCacheIterator) Error() error {
+	return nil
+}
+
+// dbCacheSnapshot defines a snapshot of the database cache and underlying
+// database at a particular point in time.
+type dbCacheSnapshot struct {
+	dbSnapshot    *leveldb.Snapshot
+	pendingKeys   *treap.Immutable
+	pendingRemove *treap.Immutable
+}
+
+// Has returns whether or not the passed key exists.
+func (snap *dbCacheSnapshot) Has(key []byte) bool {
+	// Check the cached entries first.
+	if snap.pendingRemove.Has(key) {
+		return false
+	}
+	if snap.pendingKeys.Has(key) {
+		return true
+	}
+
+	// Consult the database.
+	hasKey, _ := snap.dbSnapshot.Has(key, nil)
+	return hasKey
+}
+
+// Get returns the value for the passed key.  The function will return nil when
+// the key does not exist.
+func (snap *dbCacheSnapshot) Get(key []byte) []byte {
+	// Check the cached entries first.
+	if snap.pendingRemove.Has(key) {
+		return nil
+	}
+	if value := snap.pendingKeys.Get(key); value != nil {
+		return value
+	}
+
+	// Consult the database.
+	value, err := snap.dbSnapshot.Get(key, nil)
+	if err != nil {
+		return nil
+	}
+	return value
+}
+
+// Release releases the snapshot.
+func (snap *dbCacheSnapshot) Release() {
+	snap.dbSnapshot.Release()
+	snap.pendingKeys = nil
+	snap.pendingRemove = nil
+}
+
+// NewIterator returns a new iterator for the snapshot.  The newly returned
+// iterator is not pointing to a valid item until a call to one of the methods
+// to position it is made.
+//
+// The slice parameter allows the iterator to be limited to a range of keys.
+// The start key is inclusive and the limit key is exclusive.  Either or both
+// can be nil if the functionality is not desired.
+func (snap *dbCacheSnapshot) NewIterator(slice *util.Range) *dbCacheIterator {
+	return &dbCacheIterator{
+		dbIter:        snap.dbSnapshot.NewIterator(slice, nil),
+		cacheIter:     newLdbCacheIter(snap, slice),
+		cacheSnapshot: snap,
+	}
+}
+
+// dbCache provides a database cache layer backed by an underlying database.  It
+// allows a maximum cache size and flush interval to be specified such that the
+// cache is flushed to the database when the cache size exceeds the maximum
+// configured value or it has been longer than the configured interval since the
+// last flush.  This effectively provides transaction batching so that callers
+// can commit transactions at will without incurring large performance hits due
+// to frequent disk syncs.
+type dbCache struct {
+	// ldb is the underlying leveldb DB for metadata.
+	ldb *leveldb.DB
+
+	// store is used to sync blocks to flat files.
+	store *blockStore
+
+	// The following fields are related to flushing the cache to persistent
+	// storage.  Note that all flushing is performed in an opportunistic
+	// fashion.  This means that it is only flushed during a transaction or
+	// when the database cache is closed.
+	//
+	// maxSize is the maximum size threshold the cache can grow to before
+	// it is flushed.
+	//
+	// flushInterval is the threshold interval of time that is allowed to
+	// pass before the cache is flushed.
+	//
+	// lastFlush is the time the cache was last flushed.  It is used in
+	// conjunction with the current time and the flush interval.
+	//
+	// NOTE: These flush related fields are protected by the database write
+	// lock.
+	maxSize       uint64
+	flushInterval time.Duration
+	lastFlush     time.Time
+
+	// The following fields hold the keys that need to be stored or deleted
+	// from the underlying database once the cache is full, enough time has
+	// passed, or when the database is shutting down.  Note that these are
+	// stored using immutable treaps to support O(1) MVCC snapshots against
+	// the cached data.  The cacheLock is used to protect concurrent access
+	// for cache updates and snapshots.
+	cacheLock    sync.RWMutex
+	cachedKeys   *treap.Immutable
+	cachedRemove *treap.Immutable
+}
+
+// Snapshot returns a snapshot of the database cache and underlying database at
+// a particular point in time.
+//
+// The snapshot must be released after use by calling Release.
+func (c *dbCache) Snapshot() (*dbCacheSnapshot, error) {
+	dbSnapshot, err := c.ldb.GetSnapshot()
+	if err != nil {
+		str := "failed to open transaction"
+		return nil, convertErr(str, err)
+	}
+
+	// Since the cached keys to be added and removed use an immutable treap,
+	// a snapshot is simply obtaining the root of the tree under the lock
+	// which is used to atomically swap the root.
+	c.cacheLock.RLock()
+	cacheSnapshot := &dbCacheSnapshot{
+		dbSnapshot:    dbSnapshot,
+		pendingKeys:   c.cachedKeys,
+		pendingRemove: c.cachedRemove,
+	}
+	c.cacheLock.RUnlock()
+	return cacheSnapshot, nil
+}
+
+// updateDB invokes the passed function in the context of a managed leveldb
+// transaction.  Any errors returned from the user-supplied function will cause
+// the transaction to be rolled back and are returned from this function.
+// Otherwise, the transaction is committed when the user-supplied function
+// returns a nil error.
+func (c *dbCache) updateDB(fn func(ldbTx *leveldb.Transaction) error) error {
+	// Start a leveldb transaction.
+	ldbTx, err := c.ldb.OpenTransaction()
+	if err != nil {
+		return convertErr("failed to open ldb transaction", err)
+	}
+
+	if err := fn(ldbTx); err != nil {
+		ldbTx.Discard()
+		return err
+	}
+
+	// Commit the leveldb transaction and convert any errors as needed.
+	if err := ldbTx.Commit(); err != nil {
+		return convertErr("failed to commit leveldb transaction", err)
+	}
+	return nil
+}
+
+// TreapForEacher is an interface which allows iteration of a treap in ascending
+// order using a user-supplied callback for each key/value pair.  It mainly
+// exists so both mutable and immutable treaps can be atomically committed to
+// the database with the same function.
+type TreapForEacher interface {
+	ForEach(func(k, v []byte) bool)
+}
+
+// commitTreaps atomically commits all of the passed pending add/update/remove
+// updates to the underlying database.
+func (c *dbCache) commitTreaps(pendingKeys, pendingRemove TreapForEacher) error {
+	// Perform all leveldb updates using an atomic transaction.
+	return c.updateDB(func(ldbTx *leveldb.Transaction) error {
+		var innerErr error
+		pendingKeys.ForEach(func(k, v []byte) bool {
+			if dbErr := ldbTx.Put(k, v, nil); dbErr != nil {
+				str := fmt.Sprintf("failed to put key %q to "+
+					"ldb transaction", k)
+				innerErr = convertErr(str, dbErr)
+				return false
+			}
+			return true
+		})
+		if innerErr != nil {
+			return innerErr
+		}
+
+		pendingRemove.ForEach(func(k, v []byte) bool {
+			if dbErr := ldbTx.Delete(k, nil); dbErr != nil {
+				str := fmt.Sprintf("failed to delete "+
+					"key %q from ldb transaction",
+					k)
+				innerErr = convertErr(str, dbErr)
+				return false
+			}
+			return true
+		})
+		return innerErr
+	})
+}
+
+// flush flushes the database cache to persistent storage.  This involes syncing
+// the block store and replaying all transactions that have been applied to the
+// cache to the underlying database.
+//
+// This function MUST be called with the database write lock held.
+func (c *dbCache) flush() error {
+	c.lastFlush = time.Now()
+
+	// Sync the current write file associated with the block store.  This is
+	// necessary before writing the metadata to prevent the case where the
+	// metadata contains information about a block which actually hasn't
+	// been written yet in unexpected shutdown scenarios.
+	if err := c.store.syncBlocks(); err != nil {
+		return err
+	}
+
+	// Since the cached keys to be added and removed use an immutable treap,
+	// a snapshot is simply obtaining the root of the tree under the lock
+	// which is used to atomically swap the root.
+	c.cacheLock.RLock()
+	cachedKeys := c.cachedKeys
+	cachedRemove := c.cachedRemove
+	c.cacheLock.RUnlock()
+
+	// Nothing to do if there is no data to flush.
+	if cachedKeys.Len() == 0 && cachedRemove.Len() == 0 {
+		return nil
+	}
+
+	// Perform all leveldb updates using an atomic transaction.
+	if err := c.commitTreaps(cachedKeys, cachedRemove); err != nil {
+		return err
+	}
+
+	// Clear the cache since it has been flushed.
+	c.cacheLock.Lock()
+	c.cachedKeys = treap.NewImmutable()
+	c.cachedRemove = treap.NewImmutable()
+	c.cacheLock.Unlock()
+
+	return nil
+}
+
+// needsFlush returns whether or not the database cache needs to be flushed to
+// persistent storage based on its current size, whether or not adding all of
+// the entries in the passed database transaction would cause it to exceed the
+// configured limit, and how much time has elapsed since the last time the cache
+// was flushed.
+//
+// This function MUST be called with the database write lock held.
+func (c *dbCache) needsFlush(tx *transaction) bool {
+	// A flush is needed when more time has elapsed than the configured
+	// flush interval.
+	if time.Since(c.lastFlush) > c.flushInterval {
+		return true
+	}
+
+	// A flush is needed when the size of the database cache exceeds the
+	// specified max cache size.  The total calculated size is multiplied by
+	// 1.5 here to account for additional memory consumption that will be
+	// needed during the flush as well as old nodes in the cache that are
+	// referenced by the snapshot used by the transaction.
+	snap := tx.snapshot
+	totalSize := snap.pendingKeys.Size() + snap.pendingRemove.Size()
+	totalSize = uint64(float64(totalSize) * 1.5)
+	return totalSize > c.maxSize
+}
+
+// commitTx atomically adds all of the pending keys to add and remove into the
+// database cache.  When adding the pending keys would cause the size of the
+// cache to exceed the max cache size, or the time since the last flush exceeds
+// the configured flush interval, the cache will be flushed to the underlying
+// persistent database.
+//
+// This is an atomic operation with respect to the cache in that either all of
+// the pending keys to add and remove in the transaction will be applied or none
+// of them will.
+//
+// The database cache itself might be flushed to the underlying persistent
+// database even if the transaction fails to apply, but it will only be the
+// state of the cache without the transaction applied.
+//
+// This function MUST be called during a database write transaction which in
+// turn implies the database write lock will be held.
+func (c *dbCache) commitTx(tx *transaction) error {
+	// Flush the cache and write the current transaction directly to the
+	// database if a flush is needed.
+	if c.needsFlush(tx) {
+		if err := c.flush(); err != nil {
+			return err
+		}
+
+		// Perform all leveldb updates using an atomic transaction.
+		err := c.commitTreaps(tx.pendingKeys, tx.pendingRemove)
+		if err != nil {
+			return err
+		}
+
+		// Clear the transaction entries since they have been committed.
+		tx.pendingKeys = nil
+		tx.pendingRemove = nil
+		return nil
+	}
+
+	// At this point a database flush is not needed, so atomically commit
+	// the transaction to the cache.
+
+	// Since the cached keys to be added and removed use an immutable treap,
+	// a snapshot is simply obtaining the root of the tree under the lock
+	// which is used to atomically swap the root.
+	c.cacheLock.RLock()
+	newCachedKeys := c.cachedKeys
+	newCachedRemove := c.cachedRemove
+	c.cacheLock.RUnlock()
+
+	// Apply every key to add in the database transaction to the cache.
+	tx.pendingKeys.ForEach(func(k, v []byte) bool {
+		newCachedRemove = newCachedRemove.Delete(k)
+		newCachedKeys = newCachedKeys.Put(k, v)
+		return true
+	})
+	tx.pendingKeys = nil
+
+	// Apply every key to remove in the database transaction to the cache.
+	tx.pendingRemove.ForEach(func(k, v []byte) bool {
+		newCachedKeys = newCachedKeys.Delete(k)
+		newCachedRemove = newCachedRemove.Put(k, nil)
+		return true
+	})
+	tx.pendingRemove = nil
+
+	// Atomically replace the immutable treaps which hold the cached keys to
+	// add and delete.
+	c.cacheLock.Lock()
+	c.cachedKeys = newCachedKeys
+	c.cachedRemove = newCachedRemove
+	c.cacheLock.Unlock()
+	return nil
+}
+
+// Close cleanly shuts down the database cache by syncing all data and closing
+// the underlying leveldb database.
+//
+// This function MUST be called with the database write lock held.
+func (c *dbCache) Close() error {
+	// Flush any outstanding cached entries to disk.
+	if err := c.flush(); err != nil {
+		// Even if there is an error while flushing, attempt to close
+		// the underlying database.  The error is ignored since it would
+		// mask the flush error.
+		_ = c.ldb.Close()
+		return err
+	}
+
+	// Close the underlying leveldb database.
+	if err := c.ldb.Close(); err != nil {
+		str := "failed to close underlying leveldb database"
+		return convertErr(str, err)
+	}
+
+	return nil
+}
+
+// newDbCache returns a new database cache instance backed by the provided
+// leveldb instance.  The cache will be flushed to leveldb when the max size
+// exceeds the provided value or it has been longer than the provided interval
+// since the last flush.
+func newDbCache(ldb *leveldb.DB, store *blockStore, maxSize uint64, flushIntervalSecs uint32) *dbCache {
+	return &dbCache{
+		ldb:           ldb,
+		store:         store,
+		maxSize:       maxSize,
+		flushInterval: time.Second * time.Duration(flushIntervalSecs),
+		lastFlush:     time.Now(),
+		cachedKeys:    treap.NewImmutable(),
+		cachedRemove:  treap.NewImmutable(),
+	}
+}

vendor/github.com/btcsuite/btcd/database/ffldb/doc.go

@@ -0,0 +1,29 @@
+// Copyright (c) 2015-2016 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+/*
+Package ffldb implements a driver for the database package that uses leveldb
+for the backing metadata and flat files for block storage.
+
+This driver is the recommended driver for use with btcd.  It makes use leveldb
+for the metadata, flat files for block storage, and checksums in key areas to
+ensure data integrity.
+
+Usage
+
+This package is a driver to the database package and provides the database type
+of "ffldb".  The parameters the Open and Create functions take are the
+database path as a string and the block network:
+
+	db, err := database.Open("ffldb", "path/to/database", wire.MainNet)
+	if err != nil {
+		// Handle error
+	}
+
+	db, err := database.Create("ffldb", "path/to/database", wire.MainNet)
+	if err != nil {
+		// Handle error
+	}
+*/
+package ffldb

vendor/github.com/btcsuite/btcd/database/ffldb/driver.go

@@ -0,0 +1,84 @@
+// Copyright (c) 2015-2016 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package ffldb
+
+import (
+	"fmt"
+
+	"github.com/btcsuite/btcd/database"
+	"github.com/btcsuite/btcd/wire"
+	"github.com/btcsuite/btclog"
+)
+
+var log = btclog.Disabled
+
+const (
+	dbType = "ffldb"
+)
+
+// parseArgs parses the arguments from the database Open/Create methods.
+func parseArgs(funcName string, args ...interface{}) (string, wire.BitcoinNet, error) {
+	if len(args) != 2 {
+		return "", 0, fmt.Errorf("invalid arguments to %s.%s -- "+
+			"expected database path and block network", dbType,
+			funcName)
+	}
+
+	dbPath, ok := args[0].(string)
+	if !ok {
+		return "", 0, fmt.Errorf("first argument to %s.%s is invalid -- "+
+			"expected database path string", dbType, funcName)
+	}
+
+	network, ok := args[1].(wire.BitcoinNet)
+	if !ok {
+		return "", 0, fmt.Errorf("second argument to %s.%s is invalid -- "+
+			"expected block network", dbType, funcName)
+	}
+
+	return dbPath, network, nil
+}
+
+// openDBDriver is the callback provided during driver registration that opens
+// an existing database for use.
+func openDBDriver(args ...interface{}) (database.DB, error) {
+	dbPath, network, err := parseArgs("Open", args...)
+	if err != nil {
+		return nil, err
+	}
+
+	return openDB(dbPath, network, false)
+}
+
+// createDBDriver is the callback provided during driver registration that
+// creates, initializes, and opens a database for use.
+func createDBDriver(args ...interface{}) (database.DB, error) {
+	dbPath, network, err := parseArgs("Create", args...)
+	if err != nil {
+		return nil, err
+	}
+
+	return openDB(dbPath, network, true)
+}
+
+// useLogger is the callback provided during driver registration that sets the
+// current logger to the provided one.
+func useLogger(logger btclog.Logger) {
+	log = logger
+}
+
+func init() {
+	// Register the driver.
+	driver := database.Driver{
+		DbType:    dbType,
+		Create:    createDBDriver,
+		Open:      openDBDriver,
+		UseLogger: useLogger,
+	}
+	if err := database.RegisterDriver(driver); err != nil {
+		panic(fmt.Sprintf("Failed to regiser database driver '%s': %v",
+			dbType, err))
+	}
+}

vendor/github.com/btcsuite/btcd/database/ffldb/driver_test.go

@@ -0,0 +1,288 @@
+// Copyright (c) 2015-2016 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package ffldb_test
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"reflect"
+	"runtime"
+	"testing"
+
+	"github.com/btcsuite/btcd/chaincfg"
+	"github.com/btcsuite/btcd/database"
+	"github.com/btcsuite/btcd/database/ffldb"
+	"github.com/btcsuite/btcutil"
+)
+
+// dbType is the database type name for this driver.
+const dbType = "ffldb"
+
+// TestCreateOpenFail ensures that errors related to creating and opening a
+// database are handled properly.
+func TestCreateOpenFail(t *testing.T) {
+	t.Parallel()
+
+	// Ensure that attempting to open a database that doesn't exist returns
+	// the expected error.
+	wantErrCode := database.ErrDbDoesNotExist
+	_, err := database.Open(dbType, "noexist", blockDataNet)
+	if !checkDbError(t, "Open", err, wantErrCode) {
+		return
+	}
+
+	// Ensure that attempting to open a database with the wrong number of
+	// parameters returns the expected error.
+	wantErr := fmt.Errorf("invalid arguments to %s.Open -- expected "+
+		"database path and block network", dbType)
+	_, err = database.Open(dbType, 1, 2, 3)
+	if err.Error() != wantErr.Error() {
+		t.Errorf("Open: did not receive expected error - got %v, "+
+			"want %v", err, wantErr)
+		return
+	}
+
+	// Ensure that attempting to open a database with an invalid type for
+	// the first parameter returns the expected error.
+	wantErr = fmt.Errorf("first argument to %s.Open is invalid -- "+
+		"expected database path string", dbType)
+	_, err = database.Open(dbType, 1, blockDataNet)
+	if err.Error() != wantErr.Error() {
+		t.Errorf("Open: did not receive expected error - got %v, "+
+			"want %v", err, wantErr)
+		return
+	}
+
+	// Ensure that attempting to open a database with an invalid type for
+	// the second parameter returns the expected error.
+	wantErr = fmt.Errorf("second argument to %s.Open is invalid -- "+
+		"expected block network", dbType)
+	_, err = database.Open(dbType, "noexist", "invalid")
+	if err.Error() != wantErr.Error() {
+		t.Errorf("Open: did not receive expected error - got %v, "+
+			"want %v", err, wantErr)
+		return
+	}
+
+	// Ensure that attempting to create a database with the wrong number of
+	// parameters returns the expected error.
+	wantErr = fmt.Errorf("invalid arguments to %s.Create -- expected "+
+		"database path and block network", dbType)
+	_, err = database.Create(dbType, 1, 2, 3)
+	if err.Error() != wantErr.Error() {
+		t.Errorf("Create: did not receive expected error - got %v, "+
+			"want %v", err, wantErr)
+		return
+	}
+
+	// Ensure that attempting to create a database with an invalid type for
+	// the first parameter returns the expected error.
+	wantErr = fmt.Errorf("first argument to %s.Create is invalid -- "+
+		"expected database path string", dbType)
+	_, err = database.Create(dbType, 1, blockDataNet)
+	if err.Error() != wantErr.Error() {
+		t.Errorf("Create: did not receive expected error - got %v, "+
+			"want %v", err, wantErr)
+		return
+	}
+
+	// Ensure that attempting to create a database with an invalid type for
+	// the second parameter returns the expected error.
+	wantErr = fmt.Errorf("second argument to %s.Create is invalid -- "+
+		"expected block network", dbType)
+	_, err = database.Create(dbType, "noexist", "invalid")
+	if err.Error() != wantErr.Error() {
+		t.Errorf("Create: did not receive expected error - got %v, "+
+			"want %v", err, wantErr)
+		return
+	}
+
+	// Ensure operations against a closed database return the expected
+	// error.
+	dbPath := filepath.Join(os.TempDir(), "ffldb-createfail")
+	_ = os.RemoveAll(dbPath)
+	db, err := database.Create(dbType, dbPath, blockDataNet)
+	if err != nil {
+		t.Errorf("Create: unexpected error: %v", err)
+		return
+	}
+	defer os.RemoveAll(dbPath)
+	db.Close()
+
+	wantErrCode = database.ErrDbNotOpen
+	err = db.View(func(tx database.Tx) error {
+		return nil
+	})
+	if !checkDbError(t, "View", err, wantErrCode) {
+		return
+	}
+
+	wantErrCode = database.ErrDbNotOpen
+	err = db.Update(func(tx database.Tx) error {
+		return nil
+	})
+	if !checkDbError(t, "Update", err, wantErrCode) {
+		return
+	}
+
+	wantErrCode = database.ErrDbNotOpen
+	_, err = db.Begin(false)
+	if !checkDbError(t, "Begin(false)", err, wantErrCode) {
+		return
+	}
+
+	wantErrCode = database.ErrDbNotOpen
+	_, err = db.Begin(true)
+	if !checkDbError(t, "Begin(true)", err, wantErrCode) {
+		return
+	}
+
+	wantErrCode = database.ErrDbNotOpen
+	err = db.Close()
+	if !checkDbError(t, "Close", err, wantErrCode) {
+		return
+	}
+}
+
+// TestPersistence ensures that values stored are still valid after closing and
+// reopening the database.
+func TestPersistence(t *testing.T) {
+	t.Parallel()
+
+	// Create a new database to run tests against.
+	dbPath := filepath.Join(os.TempDir(), "ffldb-persistencetest")
+	_ = os.RemoveAll(dbPath)
+	db, err := database.Create(dbType, dbPath, blockDataNet)
+	if err != nil {
+		t.Errorf("Failed to create test database (%s) %v", dbType, err)
+		return
+	}
+	defer os.RemoveAll(dbPath)
+	defer db.Close()
+
+	// Create a bucket, put some values into it, and store a block so they
+	// can be tested for existence on re-open.
+	bucket1Key := []byte("bucket1")
+	storeValues := map[string]string{
+		"b1key1": "foo1",
+		"b1key2": "foo2",
+		"b1key3": "foo3",
+	}
+	genesisBlock := btcutil.NewBlock(chaincfg.MainNetParams.GenesisBlock)
+	genesisHash := chaincfg.MainNetParams.GenesisHash
+	err = db.Update(func(tx database.Tx) error {
+		metadataBucket := tx.Metadata()
+		if metadataBucket == nil {
+			return fmt.Errorf("Metadata: unexpected nil bucket")
+		}
+
+		bucket1, err := metadataBucket.CreateBucket(bucket1Key)
+		if err != nil {
+			return fmt.Errorf("CreateBucket: unexpected error: %v",
+				err)
+		}
+
+		for k, v := range storeValues {
+			err := bucket1.Put([]byte(k), []byte(v))
+			if err != nil {
+				return fmt.Errorf("Put: unexpected error: %v",
+					err)
+			}
+		}
+
+		if err := tx.StoreBlock(genesisBlock); err != nil {
+			return fmt.Errorf("StoreBlock: unexpected error: %v",
+				err)
+		}
+
+		return nil
+	})
+	if err != nil {
+		t.Errorf("Update: unexpected error: %v", err)
+		return
+	}
+
+	// Close and reopen the database to ensure the values persist.
+	db.Close()
+	db, err = database.Open(dbType, dbPath, blockDataNet)
+	if err != nil {
+		t.Errorf("Failed to open test database (%s) %v", dbType, err)
+		return
+	}
+	defer db.Close()
+
+	// Ensure the values previously stored in the 3rd namespace still exist
+	// and are correct.
+	err = db.View(func(tx database.Tx) error {
+		metadataBucket := tx.Metadata()
+		if metadataBucket == nil {
+			return fmt.Errorf("Metadata: unexpected nil bucket")
+		}
+
+		bucket1 := metadataBucket.Bucket(bucket1Key)
+		if bucket1 == nil {
+			return fmt.Errorf("Bucket1: unexpected nil bucket")
+		}
+
+		for k, v := range storeValues {
+			gotVal := bucket1.Get([]byte(k))
+			if !reflect.DeepEqual(gotVal, []byte(v)) {
+				return fmt.Errorf("Get: key '%s' does not "+
+					"match expected value - got %s, want %s",
+					k, gotVal, v)
+			}
+		}
+
+		genesisBlockBytes, _ := genesisBlock.Bytes()
+		gotBytes, err := tx.FetchBlock(genesisHash)
+		if err != nil {
+			return fmt.Errorf("FetchBlock: unexpected error: %v",
+				err)
+		}
+		if !reflect.DeepEqual(gotBytes, genesisBlockBytes) {
+			return fmt.Errorf("FetchBlock: stored block mismatch")
+		}
+
+		return nil
+	})
+	if err != nil {
+		t.Errorf("View: unexpected error: %v", err)
+		return
+	}
+}
+
+// TestInterface performs all interfaces tests for this database driver.
+func TestInterface(t *testing.T) {
+	t.Parallel()
+
+	// Create a new database to run tests against.
+	dbPath := filepath.Join(os.TempDir(), "ffldb-interfacetest")
+	_ = os.RemoveAll(dbPath)
+	db, err := database.Create(dbType, dbPath, blockDataNet)
+	if err != nil {
+		t.Errorf("Failed to create test database (%s) %v", dbType, err)
+		return
+	}
+	defer os.RemoveAll(dbPath)
+	defer db.Close()
+
+	// Ensure the driver type is the expected value.
+	gotDbType := db.Type()
+	if gotDbType != dbType {
+		t.Errorf("Type: unepxected driver type - got %v, want %v",
+			gotDbType, dbType)
+		return
+	}
+
+	// Run all of the interface tests against the database.
+	runtime.GOMAXPROCS(runtime.NumCPU())
+
+	// Change the maximum file size to a small value to force multiple flat
+	// files with the test data set.
+	ffldb.TstRunWithMaxBlockFileSize(db, 2048, func() {
+		testInterface(t, db)
+	})
+}

vendor/github.com/btcsuite/btcd/database/ffldb/export_test.go

@@ -0,0 +1,26 @@
+// Copyright (c) 2015-2016 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+/*
+This test file is part of the ffldb package rather than than the ffldb_test
+package so it can bridge access to the internals to properly test cases which
+are either not possible or can't reliably be tested via the public interface.
+The functions are only exported while the tests are being run.
+*/
+
+package ffldb
+
+import "github.com/btcsuite/btcd/database"
+
+// TstRunWithMaxBlockFileSize runs the passed function with the maximum allowed
+// file size for the database set to the provided value.  The value will be set
+// back to the original value upon completion.
+func TstRunWithMaxBlockFileSize(idb database.DB, size uint32, fn func()) {
+	ffldb := idb.(*db)
+	origSize := ffldb.store.maxBlockFileSize
+
+	ffldb.store.maxBlockFileSize = size
+	fn()
+	ffldb.store.maxBlockFileSize = origSize
+}

vendor/github.com/btcsuite/btcd/database/ffldb/ldbtreapiter.go

@@ -0,0 +1,58 @@
+// Copyright (c) 2015-2016 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package ffldb
+
+import (
+	"github.com/btcsuite/btcd/database/internal/treap"
+	"github.com/btcsuite/goleveldb/leveldb/iterator"
+	"github.com/btcsuite/goleveldb/leveldb/util"
+)
+
+// ldbTreapIter wraps a treap iterator to provide the additional functionality
+// needed to satisfy the leveldb iterator.Iterator interface.
+type ldbTreapIter struct {
+	*treap.Iterator
+	tx       *transaction
+	released bool
+}
+
+// Enforce ldbTreapIter implements the leveldb iterator.Iterator interface.
+var _ iterator.Iterator = (*ldbTreapIter)(nil)
+
+// Error is only provided to satisfy the iterator interface as there are no
+// errors for this memory-only structure.
+//
+// This is part of the leveldb iterator.Iterator interface implementation.
+func (iter *ldbTreapIter) Error() error {
+	return nil
+}
+
+// SetReleaser is only provided to satisfy the iterator interface as there is no
+// need to override it.
+//
+// This is part of the leveldb iterator.Iterator interface implementation.
+func (iter *ldbTreapIter) SetReleaser(releaser util.Releaser) {
+}
+
+// Release releases the iterator by removing the underlying treap iterator from
+// the list of active iterators against the pending keys treap.
+//
+// This is part of the leveldb iterator.Iterator interface implementation.
+func (iter *ldbTreapIter) Release() {
+	if !iter.released {
+		iter.tx.removeActiveIter(iter.Iterator)
+		iter.released = true
+	}
+}
+
+// newLdbTreapIter creates a new treap iterator for the given slice against the
+// pending keys for the passed transaction and returns it wrapped in an
+// ldbTreapIter so it can be used as a leveldb iterator.  It also adds the new
+// iterator to the list of active iterators for the transaction.
+func newLdbTreapIter(tx *transaction, slice *util.Range) *ldbTreapIter {
+	iter := tx.pendingKeys.Iterator(slice.Start, slice.Limit)
+	tx.addActiveIter(iter)
+	return &ldbTreapIter{Iterator: iter, tx: tx}
+}

vendor/github.com/btcsuite/btcd/database/ffldb/mockfile_test.go

@@ -0,0 +1,163 @@
+// Copyright (c) 2015-2016 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+// This file is part of the ffldb package rather than the ffldb_test package as
+// it is part of the whitebox testing.
+
+package ffldb
+
+import (
+	"errors"
+	"io"
+	"sync"
+)
+
+// Errors used for the mock file.
+var (
+	// errMockFileClosed is used to indicate a mock file is closed.
+	errMockFileClosed = errors.New("file closed")
+
+	// errInvalidOffset is used to indicate an offset that is out of range
+	// for the file was provided.
+	errInvalidOffset = errors.New("invalid offset")
+
+	// errSyncFail is used to indicate simulated sync failure.
+	errSyncFail = errors.New("simulated sync failure")
+)
+
+// mockFile implements the filer interface and used in order to force failures
+// the database code related to reading and writing from the flat block files.
+// A maxSize of -1 is unlimited.
+type mockFile struct {
+	sync.RWMutex
+	maxSize      int64
+	data         []byte
+	forceSyncErr bool
+	closed       bool
+}
+
+// Close closes the mock file without releasing any data associated with it.
+// This allows it to be "reopened" without losing the data.
+//
+// This is part of the filer implementation.
+func (f *mockFile) Close() error {
+	f.Lock()
+	defer f.Unlock()
+
+	if f.closed {
+		return errMockFileClosed
+	}
+	f.closed = true
+	return nil
+}
+
+// ReadAt reads len(b) bytes from the mock file starting at byte offset off. It
+// returns the number of bytes read and the error, if any.  ReadAt always
+// returns a non-nil error when n < len(b). At end of file, that error is
+// io.EOF.
+//
+// This is part of the filer implementation.
+func (f *mockFile) ReadAt(b []byte, off int64) (int, error) {
+	f.RLock()
+	defer f.RUnlock()
+
+	if f.closed {
+		return 0, errMockFileClosed
+	}
+	maxSize := int64(len(f.data))
+	if f.maxSize > -1 && maxSize > f.maxSize {
+		maxSize = f.maxSize
+	}
+	if off < 0 || off > maxSize {
+		return 0, errInvalidOffset
+	}
+
+	// Limit to the max size field, if set.
+	numToRead := int64(len(b))
+	endOffset := off + numToRead
+	if endOffset > maxSize {
+		numToRead = maxSize - off
+	}
+
+	copy(b, f.data[off:off+numToRead])
+	if numToRead < int64(len(b)) {
+		return int(numToRead), io.EOF
+	}
+	return int(numToRead), nil
+}
+
+// Truncate changes the size of the mock file.
+//
+// This is part of the filer implementation.
+func (f *mockFile) Truncate(size int64) error {
+	f.Lock()
+	defer f.Unlock()
+
+	if f.closed {
+		return errMockFileClosed
+	}
+	maxSize := int64(len(f.data))
+	if f.maxSize > -1 && maxSize > f.maxSize {
+		maxSize = f.maxSize
+	}
+	if size > maxSize {
+		return errInvalidOffset
+	}
+
+	f.data = f.data[:size]
+	return nil
+}
+
+// Write writes len(b) bytes to the mock file. It returns the number of bytes
+// written and an error, if any.  Write returns a non-nil error any time
+// n != len(b).
+//
+// This is part of the filer implementation.
+func (f *mockFile) WriteAt(b []byte, off int64) (int, error) {
+	f.Lock()
+	defer f.Unlock()
+
+	if f.closed {
+		return 0, errMockFileClosed
+	}
+	maxSize := f.maxSize
+	if maxSize < 0 {
+		maxSize = 100 * 1024 // 100KiB
+	}
+	if off < 0 || off > maxSize {
+		return 0, errInvalidOffset
+	}
+
+	// Limit to the max size field, if set, and grow the slice if needed.
+	numToWrite := int64(len(b))
+	if off+numToWrite > maxSize {
+		numToWrite = maxSize - off
+	}
+	if off+numToWrite > int64(len(f.data)) {
+		newData := make([]byte, off+numToWrite)
+		copy(newData, f.data)
+		f.data = newData
+	}
+
+	copy(f.data[off:], b[:numToWrite])
+	if numToWrite < int64(len(b)) {
+		return int(numToWrite), io.EOF
+	}
+	return int(numToWrite), nil
+}
+
+// Sync doesn't do anything for mock files.  However, it will return an error if
+// the mock file's forceSyncErr flag is set.
+//
+// This is part of the filer implementation.
+func (f *mockFile) Sync() error {
+	if f.forceSyncErr {
+		return errSyncFail
+	}
+
+	return nil
+}
+
+// Ensure the mockFile type implements the filer interface.
+var _ filer = (*mockFile)(nil)

vendor/github.com/btcsuite/btcd/database/ffldb/reconcile.go

@@ -0,0 +1,117 @@
+// Copyright (c) 2015-2016 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package ffldb
+
+import (
+	"fmt"
+	"hash/crc32"
+
+	"github.com/btcsuite/btcd/database"
+)
+
+// The serialized write cursor location format is:
+//
+//  [0:4]  Block file (4 bytes)
+//  [4:8]  File offset (4 bytes)
+//  [8:12] Castagnoli CRC-32 checksum (4 bytes)
+
+// serializeWriteRow serialize the current block file and offset where new
+// will be written into a format suitable for storage into the metadata.
+func serializeWriteRow(curBlockFileNum, curFileOffset uint32) []byte {
+	var serializedRow [12]byte
+	byteOrder.PutUint32(serializedRow[0:4], curBlockFileNum)
+	byteOrder.PutUint32(serializedRow[4:8], curFileOffset)
+	checksum := crc32.Checksum(serializedRow[:8], castagnoli)
+	byteOrder.PutUint32(serializedRow[8:12], checksum)
+	return serializedRow[:]
+}
+
+// deserializeWriteRow deserializes the write cursor location stored in the
+// metadata.  Returns ErrCorruption if the checksum of the entry doesn't match.
+func deserializeWriteRow(writeRow []byte) (uint32, uint32, error) {
+	// Ensure the checksum matches.  The checksum is at the end.
+	gotChecksum := crc32.Checksum(writeRow[:8], castagnoli)
+	wantChecksumBytes := writeRow[8:12]
+	wantChecksum := byteOrder.Uint32(wantChecksumBytes)
+	if gotChecksum != wantChecksum {
+		str := fmt.Sprintf("metadata for write cursor does not match "+
+			"the expected checksum - got %d, want %d", gotChecksum,
+			wantChecksum)
+		return 0, 0, makeDbErr(database.ErrCorruption, str, nil)
+	}
+
+	fileNum := byteOrder.Uint32(writeRow[0:4])
+	fileOffset := byteOrder.Uint32(writeRow[4:8])
+	return fileNum, fileOffset, nil
+}
+
+// reconcileDB reconciles the metadata with the flat block files on disk.  It
+// will also initialize the underlying database if the create flag is set.
+func reconcileDB(pdb *db, create bool) (database.DB, error) {
+	// Perform initial internal bucket and value creation during database
+	// creation.
+	if create {
+		if err := initDB(pdb.cache.ldb); err != nil {
+			return nil, err
+		}
+	}
+
+	// Load the current write cursor position from the metadata.
+	var curFileNum, curOffset uint32
+	err := pdb.View(func(tx database.Tx) error {
+		writeRow := tx.Metadata().Get(writeLocKeyName)
+		if writeRow == nil {
+			str := "write cursor does not exist"
+			return makeDbErr(database.ErrCorruption, str, nil)
+		}
+
+		var err error
+		curFileNum, curOffset, err = deserializeWriteRow(writeRow)
+		return err
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	// When the write cursor position found by scanning the block files on
+	// disk is AFTER the position the metadata believes to be true, truncate
+	// the files on disk to match the metadata.  This can be a fairly common
+	// occurrence in unclean shutdown scenarios while the block files are in
+	// the middle of being written.  Since the metadata isn't updated until
+	// after the block data is written, this is effectively just a rollback
+	// to the known good point before the unclean shutdown.
+	wc := pdb.store.writeCursor
+	if wc.curFileNum > curFileNum || (wc.curFileNum == curFileNum &&
+		wc.curOffset > curOffset) {
+
+		log.Info("Detected unclean shutdown - Repairing...")
+		log.Debugf("Metadata claims file %d, offset %d. Block data is "+
+			"at file %d, offset %d", curFileNum, curOffset,
+			wc.curFileNum, wc.curOffset)
+		pdb.store.handleRollback(curFileNum, curOffset)
+		log.Infof("Database sync complete")
+	}
+
+	// When the write cursor position found by scanning the block files on
+	// disk is BEFORE the position the metadata believes to be true, return
+	// a corruption error.  Since sync is called after each block is written
+	// and before the metadata is updated, this should only happen in the
+	// case of missing, deleted, or truncated block files, which generally
+	// is not an easily recoverable scenario.  In the future, it might be
+	// possible to rescan and rebuild the metadata from the block files,
+	// however, that would need to happen with coordination from a higher
+	// layer since it could invalidate other metadata.
+	if wc.curFileNum < curFileNum || (wc.curFileNum == curFileNum &&
+		wc.curOffset < curOffset) {
+
+		str := fmt.Sprintf("metadata claims file %d, offset %d, but "+
+			"block data is at file %d, offset %d", curFileNum,
+			curOffset, wc.curFileNum, wc.curOffset)
+		log.Warnf("***Database corruption detected***: %v", str)
+		return nil, makeDbErr(database.ErrCorruption, str, nil)
+	}
+
+	return pdb, nil
+}

vendor/github.com/btcsuite/btcd/database/ffldb/whitebox_test.go

@@ -0,0 +1,706 @@
+// Copyright (c) 2015-2016 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+// This file is part of the ffldb package rather than the ffldb_test package as
+// it provides whitebox testing.
+
+package ffldb
+
+import (
+	"compress/bzip2"
+	"encoding/binary"
+	"fmt"
+	"hash/crc32"
+	"io"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/btcsuite/btcd/chaincfg"
+	"github.com/btcsuite/btcd/database"
+	"github.com/btcsuite/btcd/wire"
+	"github.com/btcsuite/btcutil"
+	"github.com/btcsuite/goleveldb/leveldb"
+	ldberrors "github.com/btcsuite/goleveldb/leveldb/errors"
+)
+
+var (
+	// blockDataNet is the expected network in the test block data.
+	blockDataNet = wire.MainNet
+
+	// blockDataFile is the path to a file containing the first 256 blocks
+	// of the block chain.
+	blockDataFile = filepath.Join("..", "testdata", "blocks1-256.bz2")
+
+	// errSubTestFail is used to signal that a sub test returned false.
+	errSubTestFail = fmt.Errorf("sub test failure")
+)
+
+// loadBlocks loads the blocks contained in the testdata directory and returns
+// a slice of them.
+func loadBlocks(t *testing.T, dataFile string, network wire.BitcoinNet) ([]*btcutil.Block, error) {
+	// Open the file that contains the blocks for reading.
+	fi, err := os.Open(dataFile)
+	if err != nil {
+		t.Errorf("failed to open file %v, err %v", dataFile, err)
+		return nil, err
+	}
+	defer func() {
+		if err := fi.Close(); err != nil {
+			t.Errorf("failed to close file %v %v", dataFile,
+				err)
+		}
+	}()
+	dr := bzip2.NewReader(fi)
+
+	// Set the first block as the genesis block.
+	blocks := make([]*btcutil.Block, 0, 256)
+	genesis := btcutil.NewBlock(chaincfg.MainNetParams.GenesisBlock)
+	blocks = append(blocks, genesis)
+
+	// Load the remaining blocks.
+	for height := 1; ; height++ {
+		var net uint32
+		err := binary.Read(dr, binary.LittleEndian, &net)
+		if err == io.EOF {
+			// Hit end of file at the expected offset.  No error.
+			break
+		}
+		if err != nil {
+			t.Errorf("Failed to load network type for block %d: %v",
+				height, err)
+			return nil, err
+		}
+		if net != uint32(network) {
+			t.Errorf("Block doesn't match network: %v expects %v",
+				net, network)
+			return nil, err
+		}
+
+		var blockLen uint32
+		err = binary.Read(dr, binary.LittleEndian, &blockLen)
+		if err != nil {
+			t.Errorf("Failed to load block size for block %d: %v",
+				height, err)
+			return nil, err
+		}
+
+		// Read the block.
+		blockBytes := make([]byte, blockLen)
+		_, err = io.ReadFull(dr, blockBytes)
+		if err != nil {
+			t.Errorf("Failed to load block %d: %v", height, err)
+			return nil, err
+		}
+
+		// Deserialize and store the block.
+		block, err := btcutil.NewBlockFromBytes(blockBytes)
+		if err != nil {
+			t.Errorf("Failed to parse block %v: %v", height, err)
+			return nil, err
+		}
+		blocks = append(blocks, block)
+	}
+
+	return blocks, nil
+}
+
+// checkDbError ensures the passed error is a database.Error with an error code
+// that matches the passed  error code.
+func checkDbError(t *testing.T, testName string, gotErr error, wantErrCode database.ErrorCode) bool {
+	dbErr, ok := gotErr.(database.Error)
+	if !ok {
+		t.Errorf("%s: unexpected error type - got %T, want %T",
+			testName, gotErr, database.Error{})
+		return false
+	}
+	if dbErr.ErrorCode != wantErrCode {
+		t.Errorf("%s: unexpected error code - got %s (%s), want %s",
+			testName, dbErr.ErrorCode, dbErr.Description,
+			wantErrCode)
+		return false
+	}
+
+	return true
+}
+
+// testContext is used to store context information about a running test which
+// is passed into helper functions.
+type testContext struct {
+	t            *testing.T
+	db           database.DB
+	files        map[uint32]*lockableFile
+	maxFileSizes map[uint32]int64
+	blocks       []*btcutil.Block
+}
+
+// TestConvertErr ensures the leveldb error to database error conversion works
+// as expected.
+func TestConvertErr(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		err         error
+		wantErrCode database.ErrorCode
+	}{
+		{&ldberrors.ErrCorrupted{}, database.ErrCorruption},
+		{leveldb.ErrClosed, database.ErrDbNotOpen},
+		{leveldb.ErrSnapshotReleased, database.ErrTxClosed},
+		{leveldb.ErrIterReleased, database.ErrTxClosed},
+	}
+
+	for i, test := range tests {
+		gotErr := convertErr("test", test.err)
+		if gotErr.ErrorCode != test.wantErrCode {
+			t.Errorf("convertErr #%d unexpected error - got %v, "+
+				"want %v", i, gotErr.ErrorCode, test.wantErrCode)
+			continue
+		}
+	}
+}
+
+// TestCornerCases ensures several corner cases which can happen when opening
+// a database and/or block files work as expected.
+func TestCornerCases(t *testing.T) {
+	t.Parallel()
+
+	// Create a file at the datapase path to force the open below to fail.
+	dbPath := filepath.Join(os.TempDir(), "ffldb-errors")
+	_ = os.RemoveAll(dbPath)
+	fi, err := os.Create(dbPath)
+	if err != nil {
+		t.Errorf("os.Create: unexpected error: %v", err)
+		return
+	}
+	fi.Close()
+
+	// Ensure creating a new database fails when a file exists where a
+	// directory is needed.
+	testName := "openDB: fail due to file at target location"
+	wantErrCode := database.ErrDriverSpecific
+	idb, err := openDB(dbPath, blockDataNet, true)
+	if !checkDbError(t, testName, err, wantErrCode) {
+		if err == nil {
+			idb.Close()
+		}
+		_ = os.RemoveAll(dbPath)
+		return
+	}
+
+	// Remove the file and create the database to run tests against.  It
+	// should be successful this time.
+	_ = os.RemoveAll(dbPath)
+	idb, err = openDB(dbPath, blockDataNet, true)
+	if err != nil {
+		t.Errorf("openDB: unexpected error: %v", err)
+		return
+	}
+	defer os.RemoveAll(dbPath)
+	defer idb.Close()
+
+	// Ensure attempting to write to a file that can't be created returns
+	// the expected error.
+	testName = "writeBlock: open file failure"
+	filePath := blockFilePath(dbPath, 0)
+	if err := os.Mkdir(filePath, 0755); err != nil {
+		t.Errorf("os.Mkdir: unexpected error: %v", err)
+		return
+	}
+	store := idb.(*db).store
+	_, err = store.writeBlock([]byte{0x00})
+	if !checkDbError(t, testName, err, database.ErrDriverSpecific) {
+		return
+	}
+	_ = os.RemoveAll(filePath)
+
+	// Close the underlying leveldb database out from under the database.
+	ldb := idb.(*db).cache.ldb
+	ldb.Close()
+
+	// Ensure initilization errors in the underlying database work as
+	// expected.
+	testName = "initDB: reinitialization"
+	wantErrCode = database.ErrDbNotOpen
+	err = initDB(ldb)
+	if !checkDbError(t, testName, err, wantErrCode) {
+		return
+	}
+
+	// Ensure the View handles errors in the underlying leveldb database
+	// properly.
+	testName = "View: underlying leveldb error"
+	wantErrCode = database.ErrDbNotOpen
+	err = idb.View(func(tx database.Tx) error {
+		return nil
+	})
+	if !checkDbError(t, testName, err, wantErrCode) {
+		return
+	}
+
+	// Ensure the Update handles errors in the underlying leveldb database
+	// properly.
+	testName = "Update: underlying leveldb error"
+	err = idb.Update(func(tx database.Tx) error {
+		return nil
+	})
+	if !checkDbError(t, testName, err, wantErrCode) {
+		return
+	}
+}
+
+// resetDatabase removes everything from the opened database associated with the
+// test context including all metadata and the mock files.
+func resetDatabase(tc *testContext) bool {
+	// Reset the metadata.
+	err := tc.db.Update(func(tx database.Tx) error {
+		// Remove all the keys using a cursor while also generating a
+		// list of buckets.  It's not safe to remove keys during ForEach
+		// iteration nor is it safe to remove buckets during cursor
+		// iteration, so this dual approach is needed.
+		var bucketNames [][]byte
+		cursor := tx.Metadata().Cursor()
+		for ok := cursor.First(); ok; ok = cursor.Next() {
+			if cursor.Value() != nil {
+				if err := cursor.Delete(); err != nil {
+					return err
+				}
+			} else {
+				bucketNames = append(bucketNames, cursor.Key())
+			}
+		}
+
+		// Remove the buckets.
+		for _, k := range bucketNames {
+			if err := tx.Metadata().DeleteBucket(k); err != nil {
+				return err
+			}
+		}
+
+		_, err := tx.Metadata().CreateBucket(blockIdxBucketName)
+		return err
+	})
+	if err != nil {
+		tc.t.Errorf("Update: unexpected error: %v", err)
+		return false
+	}
+
+	// Reset the mock files.
+	store := tc.db.(*db).store
+	wc := store.writeCursor
+	wc.curFile.Lock()
+	if wc.curFile.file != nil {
+		wc.curFile.file.Close()
+		wc.curFile.file = nil
+	}
+	wc.curFile.Unlock()
+	wc.Lock()
+	wc.curFileNum = 0
+	wc.curOffset = 0
+	wc.Unlock()
+	tc.files = make(map[uint32]*lockableFile)
+	tc.maxFileSizes = make(map[uint32]int64)
+	return true
+}
+
+// testWriteFailures tests various failures paths when writing to the block
+// files.
+func testWriteFailures(tc *testContext) bool {
+	if !resetDatabase(tc) {
+		return false
+	}
+
+	// Ensure file sync errors during flush return the expected error.
+	store := tc.db.(*db).store
+	testName := "flush: file sync failure"
+	store.writeCursor.Lock()
+	oldFile := store.writeCursor.curFile
+	store.writeCursor.curFile = &lockableFile{
+		file: &mockFile{forceSyncErr: true, maxSize: -1},
+	}
+	store.writeCursor.Unlock()
+	err := tc.db.(*db).cache.flush()
+	if !checkDbError(tc.t, testName, err, database.ErrDriverSpecific) {
+		return false
+	}
+	store.writeCursor.Lock()
+	store.writeCursor.curFile = oldFile
+	store.writeCursor.Unlock()
+
+	// Force errors in the various error paths when writing data by using
+	// mock files with a limited max size.
+	block0Bytes, _ := tc.blocks[0].Bytes()
+	tests := []struct {
+		fileNum uint32
+		maxSize int64
+	}{
+		// Force an error when writing the network bytes.
+		{fileNum: 0, maxSize: 2},
+
+		// Force an error when writing the block size.
+		{fileNum: 0, maxSize: 6},
+
+		// Force an error when writing the block.
+		{fileNum: 0, maxSize: 17},
+
+		// Force an error when writing the checksum.
+		{fileNum: 0, maxSize: int64(len(block0Bytes)) + 10},
+
+		// Force an error after writing enough blocks for force multiple
+		// files.
+		{fileNum: 15, maxSize: 1},
+	}
+
+	for i, test := range tests {
+		if !resetDatabase(tc) {
+			return false
+		}
+
+		// Ensure storing the specified number of blocks using a mock
+		// file that fails the write fails when the transaction is
+		// committed, not when the block is stored.
+		tc.maxFileSizes = map[uint32]int64{test.fileNum: test.maxSize}
+		err := tc.db.Update(func(tx database.Tx) error {
+			for i, block := range tc.blocks {
+				err := tx.StoreBlock(block)
+				if err != nil {
+					tc.t.Errorf("StoreBlock (%d): unexpected "+
+						"error: %v", i, err)
+					return errSubTestFail
+				}
+			}
+
+			return nil
+		})
+		testName := fmt.Sprintf("Force update commit failure - test "+
+			"%d, fileNum %d, maxsize %d", i, test.fileNum,
+			test.maxSize)
+		if !checkDbError(tc.t, testName, err, database.ErrDriverSpecific) {
+			tc.t.Errorf("%v", err)
+			return false
+		}
+
+		// Ensure the commit rollback removed all extra files and data.
+		if len(tc.files) != 1 {
+			tc.t.Errorf("Update rollback: new not removed - want "+
+				"1 file, got %d", len(tc.files))
+			return false
+		}
+		if _, ok := tc.files[0]; !ok {
+			tc.t.Error("Update rollback: file 0 does not exist")
+			return false
+		}
+		file := tc.files[0].file.(*mockFile)
+		if len(file.data) != 0 {
+			tc.t.Errorf("Update rollback: file did not truncate - "+
+				"want len 0, got len %d", len(file.data))
+			return false
+		}
+	}
+
+	return true
+}
+
+// testBlockFileErrors ensures the database returns expected errors with various
+// file-related issues such as closed and missing files.
+func testBlockFileErrors(tc *testContext) bool {
+	if !resetDatabase(tc) {
+		return false
+	}
+
+	// Ensure errors in blockFile and openFile when requesting invalid file
+	// numbers.
+	store := tc.db.(*db).store
+	testName := "blockFile invalid file open"
+	_, err := store.blockFile(^uint32(0))
+	if !checkDbError(tc.t, testName, err, database.ErrDriverSpecific) {
+		return false
+	}
+	testName = "openFile invalid file open"
+	_, err = store.openFile(^uint32(0))
+	if !checkDbError(tc.t, testName, err, database.ErrDriverSpecific) {
+		return false
+	}
+
+	// Insert the first block into the mock file.
+	err = tc.db.Update(func(tx database.Tx) error {
+		err := tx.StoreBlock(tc.blocks[0])
+		if err != nil {
+			tc.t.Errorf("StoreBlock: unexpected error: %v", err)
+			return errSubTestFail
+		}
+
+		return nil
+	})
+	if err != nil {
+		if err != errSubTestFail {
+			tc.t.Errorf("Update: unexpected error: %v", err)
+		}
+		return false
+	}
+
+	// Ensure errors in readBlock and readBlockRegion when requesting a file
+	// number that doesn't exist.
+	block0Hash := tc.blocks[0].Hash()
+	testName = "readBlock invalid file number"
+	invalidLoc := blockLocation{
+		blockFileNum: ^uint32(0),
+		blockLen:     80,
+	}
+	_, err = store.readBlock(block0Hash, invalidLoc)
+	if !checkDbError(tc.t, testName, err, database.ErrDriverSpecific) {
+		return false
+	}
+	testName = "readBlockRegion invalid file number"
+	_, err = store.readBlockRegion(invalidLoc, 0, 80)
+	if !checkDbError(tc.t, testName, err, database.ErrDriverSpecific) {
+		return false
+	}
+
+	// Close the block file out from under the database.
+	store.writeCursor.curFile.Lock()
+	store.writeCursor.curFile.file.Close()
+	store.writeCursor.curFile.Unlock()
+
+	// Ensure failures in FetchBlock and FetchBlockRegion(s) since the
+	// underlying file they need to read from has been closed.
+	err = tc.db.View(func(tx database.Tx) error {
+		testName = "FetchBlock closed file"
+		wantErrCode := database.ErrDriverSpecific
+		_, err := tx.FetchBlock(block0Hash)
+		if !checkDbError(tc.t, testName, err, wantErrCode) {
+			return errSubTestFail
+		}
+
+		testName = "FetchBlockRegion closed file"
+		regions := []database.BlockRegion{
+			{
+				Hash:   block0Hash,
+				Len:    80,
+				Offset: 0,
+			},
+		}
+		_, err = tx.FetchBlockRegion(&regions[0])
+		if !checkDbError(tc.t, testName, err, wantErrCode) {
+			return errSubTestFail
+		}
+
+		testName = "FetchBlockRegions closed file"
+		_, err = tx.FetchBlockRegions(regions)
+		if !checkDbError(tc.t, testName, err, wantErrCode) {
+			return errSubTestFail
+		}
+
+		return nil
+	})
+	if err != nil {
+		if err != errSubTestFail {
+			tc.t.Errorf("View: unexpected error: %v", err)
+		}
+		return false
+	}
+
+	return true
+}
+
+// testCorruption ensures the database returns expected errors under various
+// corruption scenarios.
+func testCorruption(tc *testContext) bool {
+	if !resetDatabase(tc) {
+		return false
+	}
+
+	// Insert the first block into the mock file.
+	err := tc.db.Update(func(tx database.Tx) error {
+		err := tx.StoreBlock(tc.blocks[0])
+		if err != nil {
+			tc.t.Errorf("StoreBlock: unexpected error: %v", err)
+			return errSubTestFail
+		}
+
+		return nil
+	})
+	if err != nil {
+		if err != errSubTestFail {
+			tc.t.Errorf("Update: unexpected error: %v", err)
+		}
+		return false
+	}
+
+	// Ensure corruption is detected by intentionally modifying the bytes
+	// stored to the mock file and reading the block.
+	block0Bytes, _ := tc.blocks[0].Bytes()
+	block0Hash := tc.blocks[0].Hash()
+	tests := []struct {
+		offset      uint32
+		fixChecksum bool
+		wantErrCode database.ErrorCode
+	}{
+		// One of the network bytes.  The checksum needs to be fixed so
+		// the invalid network is detected.
+		{2, true, database.ErrDriverSpecific},
+
+		// The same network byte, but this time don't fix the checksum
+		// to ensure the corruption is detected.
+		{2, false, database.ErrCorruption},
+
+		// One of the block length bytes.
+		{6, false, database.ErrCorruption},
+
+		// Random header byte.
+		{17, false, database.ErrCorruption},
+
+		// Random transaction byte.
+		{90, false, database.ErrCorruption},
+
+		// Random checksum byte.
+		{uint32(len(block0Bytes)) + 10, false, database.ErrCorruption},
+	}
+	err = tc.db.View(func(tx database.Tx) error {
+		data := tc.files[0].file.(*mockFile).data
+		for i, test := range tests {
+			// Corrupt the byte at the offset by a single bit.
+			data[test.offset] ^= 0x10
+
+			// Fix the checksum if requested to force other errors.
+			fileLen := len(data)
+			var oldChecksumBytes [4]byte
+			copy(oldChecksumBytes[:], data[fileLen-4:])
+			if test.fixChecksum {
+				toSum := data[:fileLen-4]
+				cksum := crc32.Checksum(toSum, castagnoli)
+				binary.BigEndian.PutUint32(data[fileLen-4:], cksum)
+			}
+
+			testName := fmt.Sprintf("FetchBlock (test #%d): "+
+				"corruption", i)
+			_, err := tx.FetchBlock(block0Hash)
+			if !checkDbError(tc.t, testName, err, test.wantErrCode) {
+				return errSubTestFail
+			}
+
+			// Reset the corrupted data back to the original.
+			data[test.offset] ^= 0x10
+			if test.fixChecksum {
+				copy(data[fileLen-4:], oldChecksumBytes[:])
+			}
+		}
+
+		return nil
+	})
+	if err != nil {
+		if err != errSubTestFail {
+			tc.t.Errorf("View: unexpected error: %v", err)
+		}
+		return false
+	}
+
+	return true
+}
+
+// TestFailureScenarios ensures several failure scenarios such as database
+// corruption, block file write failures, and rollback failures are handled
+// correctly.
+func TestFailureScenarios(t *testing.T) {
+	// Create a new database to run tests against.
+	dbPath := filepath.Join(os.TempDir(), "ffldb-failurescenarios")
+	_ = os.RemoveAll(dbPath)
+	idb, err := database.Create(dbType, dbPath, blockDataNet)
+	if err != nil {
+		t.Errorf("Failed to create test database (%s) %v", dbType, err)
+		return
+	}
+	defer os.RemoveAll(dbPath)
+	defer idb.Close()
+
+	// Create a test context to pass around.
+	tc := &testContext{
+		t:            t,
+		db:           idb,
+		files:        make(map[uint32]*lockableFile),
+		maxFileSizes: make(map[uint32]int64),
+	}
+
+	// Change the maximum file size to a small value to force multiple flat
+	// files with the test data set and replace the file-related functions
+	// to make use of mock files in memory.  This allows injection of
+	// various file-related errors.
+	store := idb.(*db).store
+	store.maxBlockFileSize = 1024 // 1KiB
+	store.openWriteFileFunc = func(fileNum uint32) (filer, error) {
+		if file, ok := tc.files[fileNum]; ok {
+			// "Reopen" the file.
+			file.Lock()
+			mock := file.file.(*mockFile)
+			mock.Lock()
+			mock.closed = false
+			mock.Unlock()
+			file.Unlock()
+			return mock, nil
+		}
+
+		// Limit the max size of the mock file as specified in the test
+		// context.
+		maxSize := int64(-1)
+		if maxFileSize, ok := tc.maxFileSizes[fileNum]; ok {
+			maxSize = int64(maxFileSize)
+		}
+		file := &mockFile{maxSize: int64(maxSize)}
+		tc.files[fileNum] = &lockableFile{file: file}
+		return file, nil
+	}
+	store.openFileFunc = func(fileNum uint32) (*lockableFile, error) {
+		// Force error when trying to open max file num.
+		if fileNum == ^uint32(0) {
+			return nil, makeDbErr(database.ErrDriverSpecific,
+				"test", nil)
+		}
+		if file, ok := tc.files[fileNum]; ok {
+			// "Reopen" the file.
+			file.Lock()
+			mock := file.file.(*mockFile)
+			mock.Lock()
+			mock.closed = false
+			mock.Unlock()
+			file.Unlock()
+			return file, nil
+		}
+		file := &lockableFile{file: &mockFile{}}
+		tc.files[fileNum] = file
+		return file, nil
+	}
+	store.deleteFileFunc = func(fileNum uint32) error {
+		if file, ok := tc.files[fileNum]; ok {
+			file.Lock()
+			file.file.Close()
+			file.Unlock()
+			delete(tc.files, fileNum)
+			return nil
+		}
+
+		str := fmt.Sprintf("file %d does not exist", fileNum)
+		return makeDbErr(database.ErrDriverSpecific, str, nil)
+	}
+
+	// Load the test blocks and save in the test context for use throughout
+	// the tests.
+	blocks, err := loadBlocks(t, blockDataFile, blockDataNet)
+	if err != nil {
+		t.Errorf("loadBlocks: Unexpected error: %v", err)
+		return
+	}
+	tc.blocks = blocks
+
+	// Test various failures paths when writing to the block files.
+	if !testWriteFailures(tc) {
+		return
+	}
+
+	// Test various file-related issues such as closed and missing files.
+	if !testBlockFileErrors(tc) {
+		return
+	}
+
+	// Test various corruption scenarios.
+	testCorruption(tc)
+}