build success

10 files changed, 669 insertions, 2457 deletions

diff --git a/core/blockchain.go b/core/blockchain.go
index b861220..82e3b6c 100644
--- a/core/blockchain.go
+++ b/core/blockchain.go
@@ -2498,6 +2498,6 @@ func (bc *BlockChain) ManualHead(hash common.Hash) error {
 	}
 	bc.chainmu.Lock()
 	defer bc.chainmu.Unlock()
-	bc.insert(block)
+	bc.writeHeadBlock(block)
 	return nil
 }
diff --git a/core/forkid/forkid.go b/core/forkid/forkid.go
new file mode 100644
index 0000000..1d6563d
--- /dev/null
+++ b/core/forkid/forkid.go
@@ -0,0 +1,258 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Package forkid implements EIP-2124 (https://eips.ethereum.org/EIPS/eip-2124).
+package forkid
+
+import (
+	"encoding/binary"
+	"errors"
+	"hash/crc32"
+	"math"
+	"math/big"
+	"reflect"
+	"strings"
+
+	"github.com/ava-labs/coreth/core/types"
+	"github.com/ava-labs/coreth/params"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/log"
+)
+
+var (
+	// ErrRemoteStale is returned by the validator if a remote fork checksum is a
+	// subset of our already applied forks, but the announced next fork block is
+	// not on our already passed chain.
+	ErrRemoteStale = errors.New("remote needs update")
+
+	// ErrLocalIncompatibleOrStale is returned by the validator if a remote fork
+	// checksum does not match any local checksum variation, signalling that the
+	// two chains have diverged in the past at some point (possibly at genesis).
+	ErrLocalIncompatibleOrStale = errors.New("local incompatible or needs update")
+)
+
+// Blockchain defines all necessary method to build a forkID.
+type Blockchain interface {
+	// Config retrieves the chain's fork configuration.
+	Config() *params.ChainConfig
+
+	// Genesis retrieves the chain's genesis block.
+	Genesis() *types.Block
+
+	// CurrentHeader retrieves the current head header of the canonical chain.
+	CurrentHeader() *types.Header
+}
+
+// ID is a fork identifier as defined by EIP-2124.
+type ID struct {
+	Hash [4]byte // CRC32 checksum of the genesis block and passed fork block numbers
+	Next uint64  // Block number of the next upcoming fork, or 0 if no forks are known
+}
+
+// Filter is a fork id filter to validate a remotely advertised ID.
+type Filter func(id ID) error
+
+// NewID calculates the Ethereum fork ID from the chain config and head.
+func NewID(chain Blockchain) ID {
+	return newID(
+		chain.Config(),
+		chain.Genesis().Hash(),
+		chain.CurrentHeader().Number.Uint64(),
+	)
+}
+
+// newID is the internal version of NewID, which takes extracted values as its
+// arguments instead of a chain. The reason is to allow testing the IDs without
+// having to simulate an entire blockchain.
+func newID(config *params.ChainConfig, genesis common.Hash, head uint64) ID {
+	// Calculate the starting checksum from the genesis hash
+	hash := crc32.ChecksumIEEE(genesis[:])
+
+	// Calculate the current fork checksum and the next fork block
+	var next uint64
+	for _, fork := range gatherForks(config) {
+		if fork <= head {
+			// Fork already passed, checksum the previous hash and the fork number
+			hash = checksumUpdate(hash, fork)
+			continue
+		}
+		next = fork
+		break
+	}
+	return ID{Hash: checksumToBytes(hash), Next: next}
+}
+
+// NewFilter creates a filter that returns if a fork ID should be rejected or not
+// based on the local chain's status.
+func NewFilter(chain Blockchain) Filter {
+	return newFilter(
+		chain.Config(),
+		chain.Genesis().Hash(),
+		func() uint64 {
+			return chain.CurrentHeader().Number.Uint64()
+		},
+	)
+}
+
+// NewStaticFilter creates a filter at block zero.
+func NewStaticFilter(config *params.ChainConfig, genesis common.Hash) Filter {
+	head := func() uint64 { return 0 }
+	return newFilter(config, genesis, head)
+}
+
+// newFilter is the internal version of NewFilter, taking closures as its arguments
+// instead of a chain. The reason is to allow testing it without having to simulate
+// an entire blockchain.
+func newFilter(config *params.ChainConfig, genesis common.Hash, headfn func() uint64) Filter {
+	// Calculate the all the valid fork hash and fork next combos
+	var (
+		forks = gatherForks(config)
+		sums  = make([][4]byte, len(forks)+1) // 0th is the genesis
+	)
+	hash := crc32.ChecksumIEEE(genesis[:])
+	sums[0] = checksumToBytes(hash)
+	for i, fork := range forks {
+		hash = checksumUpdate(hash, fork)
+		sums[i+1] = checksumToBytes(hash)
+	}
+	// Add two sentries to simplify the fork checks and don't require special
+	// casing the last one.
+	forks = append(forks, math.MaxUint64) // Last fork will never be passed
+
+	// Create a validator that will filter out incompatible chains
+	return func(id ID) error {
+		// Run the fork checksum validation ruleset:
+		//   1. If local and remote FORK_CSUM matches, compare local head to FORK_NEXT.
+		//        The two nodes are in the same fork state currently. They might know
+		//        of differing future forks, but that's not relevant until the fork
+		//        triggers (might be postponed, nodes might be updated to match).
+		//      1a. A remotely announced but remotely not passed block is already passed
+		//          locally, disconnect, since the chains are incompatible.
+		//      1b. No remotely announced fork; or not yet passed locally, connect.
+		//   2. If the remote FORK_CSUM is a subset of the local past forks and the
+		//      remote FORK_NEXT matches with the locally following fork block number,
+		//      connect.
+		//        Remote node is currently syncing. It might eventually diverge from
+		//        us, but at this current point in time we don't have enough information.
+		//   3. If the remote FORK_CSUM is a superset of the local past forks and can
+		//      be completed with locally known future forks, connect.
+		//        Local node is currently syncing. It might eventually diverge from
+		//        the remote, but at this current point in time we don't have enough
+		//        information.
+		//   4. Reject in all other cases.
+		head := headfn()
+		for i, fork := range forks {
+			// If our head is beyond this fork, continue to the next (we have a dummy
+			// fork of maxuint64 as the last item to always fail this check eventually).
+			if head > fork {
+				continue
+			}
+			// Found the first unpassed fork block, check if our current state matches
+			// the remote checksum (rule #1).
+			if sums[i] == id.Hash {
+				// Fork checksum matched, check if a remote future fork block already passed
+				// locally without the local node being aware of it (rule #1a).
+				if id.Next > 0 && head >= id.Next {
+					return ErrLocalIncompatibleOrStale
+				}
+				// Haven't passed locally a remote-only fork, accept the connection (rule #1b).
+				return nil
+			}
+			// The local and remote nodes are in different forks currently, check if the
+			// remote checksum is a subset of our local forks (rule #2).
+			for j := 0; j < i; j++ {
+				if sums[j] == id.Hash {
+					// Remote checksum is a subset, validate based on the announced next fork
+					if forks[j] != id.Next {
+						return ErrRemoteStale
+					}
+					return nil
+				}
+			}
+			// Remote chain is not a subset of our local one, check if it's a superset by
+			// any chance, signalling that we're simply out of sync (rule #3).
+			for j := i + 1; j < len(sums); j++ {
+				if sums[j] == id.Hash {
+					// Yay, remote checksum is a superset, ignore upcoming forks
+					return nil
+				}
+			}
+			// No exact, subset or superset match. We are on differing chains, reject.
+			return ErrLocalIncompatibleOrStale
+		}
+		log.Error("Impossible fork ID validation", "id", id)
+		return nil // Something's very wrong, accept rather than reject
+	}
+}
+
+// checksumUpdate calculates the next IEEE CRC32 checksum based on the previous
+// one and a fork block number (equivalent to CRC32(original-blob || fork)).
+func checksumUpdate(hash uint32, fork uint64) uint32 {
+	var blob [8]byte
+	binary.BigEndian.PutUint64(blob[:], fork)
+	return crc32.Update(hash, crc32.IEEETable, blob[:])
+}
+
+// checksumToBytes converts a uint32 checksum into a [4]byte array.
+func checksumToBytes(hash uint32) [4]byte {
+	var blob [4]byte
+	binary.BigEndian.PutUint32(blob[:], hash)
+	return blob
+}
+
+// gatherForks gathers all the known forks and creates a sorted list out of them.
+func gatherForks(config *params.ChainConfig) []uint64 {
+	// Gather all the fork block numbers via reflection
+	kind := reflect.TypeOf(params.ChainConfig{})
+	conf := reflect.ValueOf(config).Elem()
+
+	var forks []uint64
+	for i := 0; i < kind.NumField(); i++ {
+		// Fetch the next field and skip non-fork rules
+		field := kind.Field(i)
+		if !strings.HasSuffix(field.Name, "Block") {
+			continue
+		}
+		if field.Type != reflect.TypeOf(new(big.Int)) {
+			continue
+		}
+		// Extract the fork rule block number and aggregate it
+		rule := conf.Field(i).Interface().(*big.Int)
+		if rule != nil {
+			forks = append(forks, rule.Uint64())
+		}
+	}
+	// Sort the fork block numbers to permit chronological XOR
+	for i := 0; i < len(forks); i++ {
+		for j := i + 1; j < len(forks); j++ {
+			if forks[i] > forks[j] {
+				forks[i], forks[j] = forks[j], forks[i]
+			}
+		}
+	}
+	// Deduplicate block numbers applying multiple forks
+	for i := 1; i < len(forks); i++ {
+		if forks[i] == forks[i-1] {
+			forks = append(forks[:i], forks[i+1:]...)
+			i--
+		}
+	}
+	// Skip any forks in block 0, that's the genesis ruleset
+	if len(forks) > 0 && forks[0] == 0 {
+		forks = forks[1:]
+	}
+	return forks
+}
diff --git a/core/rawdb/accessors_state.go b/core/rawdb/accessors_state.go
new file mode 100644
index 0000000..6112de0
--- /dev/null
+++ b/core/rawdb/accessors_state.go
@@ -0,0 +1,96 @@
+// Copyright 2020 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package rawdb
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/log"
+)
+
+// ReadPreimage retrieves a single preimage of the provided hash.
+func ReadPreimage(db ethdb.KeyValueReader, hash common.Hash) []byte {
+	data, _ := db.Get(preimageKey(hash))
+	return data
+}
+
+// WritePreimages writes the provided set of preimages to the database.
+func WritePreimages(db ethdb.KeyValueWriter, preimages map[common.Hash][]byte) {
+	for hash, preimage := range preimages {
+		if err := db.Put(preimageKey(hash), preimage); err != nil {
+			log.Crit("Failed to store trie preimage", "err", err)
+		}
+	}
+	preimageCounter.Inc(int64(len(preimages)))
+	preimageHitCounter.Inc(int64(len(preimages)))
+}
+
+// ReadCode retrieves the contract code of the provided code hash.
+func ReadCode(db ethdb.KeyValueReader, hash common.Hash) []byte {
+	// Try with the legacy code scheme first, if not then try with current
+	// scheme. Since most of the code will be found with legacy scheme.
+	//
+	// todo(rjl493456442) change the order when we forcibly upgrade the code
+	// scheme with snapshot.
+	data, _ := db.Get(hash[:])
+	if len(data) != 0 {
+		return data
+	}
+	return ReadCodeWithPrefix(db, hash)
+}
+
+// ReadCodeWithPrefix retrieves the contract code of the provided code hash.
+// The main difference between this function and ReadCode is this function
+// will only check the existence with latest scheme(with prefix).
+func ReadCodeWithPrefix(db ethdb.KeyValueReader, hash common.Hash) []byte {
+	data, _ := db.Get(codeKey(hash))
+	return data
+}
+
+// WriteCode writes the provided contract code database.
+func WriteCode(db ethdb.KeyValueWriter, hash common.Hash, code []byte) {
+	if err := db.Put(codeKey(hash), code); err != nil {
+		log.Crit("Failed to store contract code", "err", err)
+	}
+}
+
+// DeleteCode deletes the specified contract code from the database.
+func DeleteCode(db ethdb.KeyValueWriter, hash common.Hash) {
+	if err := db.Delete(codeKey(hash)); err != nil {
+		log.Crit("Failed to delete contract code", "err", err)
+	}
+}
+
+// ReadTrieNode retrieves the trie node of the provided hash.
+func ReadTrieNode(db ethdb.KeyValueReader, hash common.Hash) []byte {
+	data, _ := db.Get(hash.Bytes())
+	return data
+}
+
+// WriteTrieNode writes the provided trie node database.
+func WriteTrieNode(db ethdb.KeyValueWriter, hash common.Hash, node []byte) {
+	if err := db.Put(hash.Bytes(), node); err != nil {
+		log.Crit("Failed to store trie node", "err", err)
+	}
+}
+
+// DeleteTrieNode deletes the specified trie node from the database.
+func DeleteTrieNode(db ethdb.KeyValueWriter, hash common.Hash) {
+	if err := db.Delete(hash.Bytes()); err != nil {
+		log.Crit("Failed to delete trie node", "err", err)
+	}
+}
diff --git a/core/rawdb/chain_iterator.go b/core/rawdb/chain_iterator.go
new file mode 100644
index 0000000..3130e92
--- /dev/null
+++ b/core/rawdb/chain_iterator.go
@@ -0,0 +1,304 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package rawdb
+
+import (
+	"runtime"
+	"sync/atomic"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/prque"
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/rlp"
+	"golang.org/x/crypto/sha3"
+)
+
+// InitDatabaseFromFreezer reinitializes an empty database from a previous batch
+// of frozen ancient blocks. The method iterates over all the frozen blocks and
+// injects into the database the block hash->number mappings.
+func InitDatabaseFromFreezer(db ethdb.Database) {
+	// If we can't access the freezer or it's empty, abort
+	frozen, err := db.Ancients()
+	if err != nil || frozen == 0 {
+		return
+	}
+	var (
+		batch  = db.NewBatch()
+		start  = time.Now()
+		logged = start.Add(-7 * time.Second) // Unindex during import is fast, don't double log
+		hash   common.Hash
+	)
+	for i := uint64(0); i < frozen; i++ {
+		// Since the freezer has all data in sequential order on a file,
+		// it would be 'neat' to read more data in one go, and let the
+		// freezerdb return N items (e.g up to 1000 items per go)
+		// That would require an API change in Ancients though
+		if h, err := db.Ancient(freezerHashTable, i); err != nil {
+			log.Crit("Failed to init database from freezer", "err", err)
+		} else {
+			hash = common.BytesToHash(h)
+		}
+		WriteHeaderNumber(batch, hash, i)
+		// If enough data was accumulated in memory or we're at the last block, dump to disk
+		if batch.ValueSize() > ethdb.IdealBatchSize {
+			if err := batch.Write(); err != nil {
+				log.Crit("Failed to write data to db", "err", err)
+			}
+			batch.Reset()
+		}
+		// If we've spent too much time already, notify the user of what we're doing
+		if time.Since(logged) > 8*time.Second {
+			log.Info("Initializing database from freezer", "total", frozen, "number", i, "hash", hash, "elapsed", common.PrettyDuration(time.Since(start)))
+			logged = time.Now()
+		}
+	}
+	if err := batch.Write(); err != nil {
+		log.Crit("Failed to write data to db", "err", err)
+	}
+	batch.Reset()
+
+	WriteHeadHeaderHash(db, hash)
+	WriteHeadFastBlockHash(db, hash)
+	log.Info("Initialized database from freezer", "blocks", frozen, "elapsed", common.PrettyDuration(time.Since(start)))
+}
+
+type blockTxHashes struct {
+	number uint64
+	hashes []common.Hash
+}
+
+// iterateTransactions iterates over all transactions in the (canon) block
+// number(s) given, and yields the hashes on a channel
+func iterateTransactions(db ethdb.Database, from uint64, to uint64, reverse bool) (chan *blockTxHashes, chan struct{}) {
+	// One thread sequentially reads data from db
+	type numberRlp struct {
+		number uint64
+		rlp    rlp.RawValue
+	}
+	if to == from {
+		return nil, nil
+	}
+	threads := to - from
+	if cpus := runtime.NumCPU(); threads > uint64(cpus) {
+		threads = uint64(cpus)
+	}
+	var (
+		rlpCh    = make(chan *numberRlp, threads*2)     // we send raw rlp over this channel
+		hashesCh = make(chan *blockTxHashes, threads*2) // send hashes over hashesCh
+		abortCh  = make(chan struct{})
+	)
+	// lookup runs in one instance
+	lookup := func() {
+		n, end := from, to
+		if reverse {
+			n, end = to-1, from-1
+		}
+		defer close(rlpCh)
+		for n != end {
+			data := ReadCanonicalBodyRLP(db, n)
+			// Feed the block to the aggregator, or abort on interrupt
+			select {
+			case rlpCh <- &numberRlp{n, data}:
+			case <-abortCh:
+				return
+			}
+			if reverse {
+				n--
+			} else {
+				n++
+			}
+		}
+	}
+	// process runs in parallel
+	nThreadsAlive := int32(threads)
+	process := func() {
+		defer func() {
+			// Last processor closes the result channel
+			if atomic.AddInt32(&nThreadsAlive, -1) == 0 {
+				close(hashesCh)
+			}
+		}()
+
+		var hasher = sha3.NewLegacyKeccak256()
+		for data := range rlpCh {
+			it, err := rlp.NewListIterator(data.rlp)
+			if err != nil {
+				log.Warn("tx iteration error", "error", err)
+				return
+			}
+			it.Next()
+			txs := it.Value()
+			txIt, err := rlp.NewListIterator(txs)
+			if err != nil {
+				log.Warn("tx iteration error", "error", err)
+				return
+			}
+			var hashes []common.Hash
+			for txIt.Next() {
+				if err := txIt.Err(); err != nil {
+					log.Warn("tx iteration error", "error", err)
+					return
+				}
+				var txHash common.Hash
+				hasher.Reset()
+				hasher.Write(txIt.Value())
+				hasher.Sum(txHash[:0])
+				hashes = append(hashes, txHash)
+			}
+			result := &blockTxHashes{
+				hashes: hashes,
+				number: data.number,
+			}
+			// Feed the block to the aggregator, or abort on interrupt
+			select {
+			case hashesCh <- result:
+			case <-abortCh:
+				return
+			}
+		}
+	}
+	go lookup() // start the sequential db accessor
+	for i := 0; i < int(threads); i++ {
+		go process()
+	}
+	return hashesCh, abortCh
+}
+
+// IndexTransactions creates txlookup indices of the specified block range.
+//
+// This function iterates canonical chain in reverse order, it has one main advantage:
+// We can write tx index tail flag periodically even without the whole indexing
+// procedure is finished. So that we can resume indexing procedure next time quickly.
+func IndexTransactions(db ethdb.Database, from uint64, to uint64) {
+	// short circuit for invalid range
+	if from >= to {
+		return
+	}
+	var (
+		hashesCh, abortCh = iterateTransactions(db, from, to, true)
+		batch             = db.NewBatch()
+		start             = time.Now()
+		logged            = start.Add(-7 * time.Second)
+		//  Since we iterate in reverse, we expect the first number to come
+		// in to be [to-1]. Therefore, setting lastNum to means that the
+		// prqueue gap-evaluation will work correctly
+		lastNum = to
+		queue   = prque.New(nil)
+		// for stats reporting
+		blocks, txs = 0, 0
+	)
+	defer close(abortCh)
+
+	for chanDelivery := range hashesCh {
+		// Push the delivery into the queue and process contiguous ranges.
+		// Since we iterate in reverse, so lower numbers have lower prio, and
+		// we can use the number directly as prio marker
+		queue.Push(chanDelivery, int64(chanDelivery.number))
+		for !queue.Empty() {
+			// If the next available item is gapped, return
+			if _, priority := queue.Peek(); priority != int64(lastNum-1) {
+				break
+			}
+			// Next block available, pop it off and index it
+			delivery := queue.PopItem().(*blockTxHashes)
+			lastNum = delivery.number
+			WriteTxLookupEntries(batch, delivery.number, delivery.hashes)
+			blocks++
+			txs += len(delivery.hashes)
+			// If enough data was accumulated in memory or we're at the last block, dump to disk
+			if batch.ValueSize() > ethdb.IdealBatchSize {
+				// Also write the tail there
+				WriteTxIndexTail(batch, lastNum)
+				if err := batch.Write(); err != nil {
+					log.Crit("Failed writing batch to db", "error", err)
+					return
+				}
+				batch.Reset()
+			}
+			// If we've spent too much time already, notify the user of what we're doing
+			if time.Since(logged) > 8*time.Second {
+				log.Info("Indexing transactions", "blocks", blocks, "txs", txs, "tail", lastNum, "total", to-from, "elapsed", common.PrettyDuration(time.Since(start)))
+				logged = time.Now()
+			}
+		}
+	}
+	if lastNum < to {
+		WriteTxIndexTail(batch, lastNum)
+		// No need to write the batch if we never entered the loop above...
+		if err := batch.Write(); err != nil {
+			log.Crit("Failed writing batch to db", "error", err)
+			return
+		}
+	}
+	log.Info("Indexed transactions", "blocks", blocks, "txs", txs, "tail", lastNum, "elapsed", common.PrettyDuration(time.Since(start)))
+}
+
+// UnindexTransactions removes txlookup indices of the specified block range.
+func UnindexTransactions(db ethdb.Database, from uint64, to uint64) {
+	// short circuit for invalid range
+	if from >= to {
+		return
+	}
+	// Write flag first and then unindex the transaction indices. Some indices
+	// will be left in the database if crash happens but it's fine.
+	WriteTxIndexTail(db, to)
+	// If only one block is unindexed, do it directly
+	//if from+1 == to {
+	//	data := ReadCanonicalBodyRLP(db, uint64(from))
+	//	DeleteTxLookupEntries(db, ReadBlock(db, ReadCanonicalHash(db, from), from))
+	//	log.Info("Unindexed transactions", "blocks", 1, "tail", to)
+	//	return
+	//}
+	// TODO @holiman, add this back (if we want it)
+	var (
+		hashesCh, abortCh = iterateTransactions(db, from, to, false)
+		batch             = db.NewBatch()
+		start             = time.Now()
+		logged            = start.Add(-7 * time.Second)
+	)
+	defer close(abortCh)
+	// Otherwise spin up the concurrent iterator and unindexer
+	blocks, txs := 0, 0
+	for delivery := range hashesCh {
+		DeleteTxLookupEntries(batch, delivery.hashes)
+		txs += len(delivery.hashes)
+		blocks++
+
+		// If enough data was accumulated in memory or we're at the last block, dump to disk
+		// A batch counts the size of deletion as '1', so we need to flush more
+		// often than that.
+		if blocks%1000 == 0 {
+			if err := batch.Write(); err != nil {
+				log.Crit("Failed writing batch to db", "error", err)
+				return
+			}
+			batch.Reset()
+		}
+		// If we've spent too much time already, notify the user of what we're doing
+		if time.Since(logged) > 8*time.Second {
+			log.Info("Unindexing transactions", "blocks", blocks, "txs", txs, "total", to-from, "elapsed", common.PrettyDuration(time.Since(start)))
+			logged = time.Now()
+		}
+	}
+	if err := batch.Write(); err != nil {
+		log.Crit("Failed writing batch to db", "error", err)
+		return
+	}
+	log.Info("Unindexed transactions", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start)))
+}
diff --git a/core/state/snapshot/difflayer_test.go b/core/state/snapshot/difflayer_test.go
deleted file mode 100644
index 31636ee..0000000
--- a/core/state/snapshot/difflayer_test.go
+++ /dev/null
@@ -1,400 +0,0 @@
-// Copyright 2019 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package snapshot
-
-import (
-	"bytes"
-	"math/rand"
-	"testing"
-
-	"github.com/VictoriaMetrics/fastcache"
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/ethdb/memorydb"
-)
-
-func copyDestructs(destructs map[common.Hash]struct{}) map[common.Hash]struct{} {
-	copy := make(map[common.Hash]struct{})
-	for hash := range destructs {
-		copy[hash] = struct{}{}
-	}
-	return copy
-}
-
-func copyAccounts(accounts map[common.Hash][]byte) map[common.Hash][]byte {
-	copy := make(map[common.Hash][]byte)
-	for hash, blob := range accounts {
-		copy[hash] = blob
-	}
-	return copy
-}
-
-func copyStorage(storage map[common.Hash]map[common.Hash][]byte) map[common.Hash]map[common.Hash][]byte {
-	copy := make(map[common.Hash]map[common.Hash][]byte)
-	for accHash, slots := range storage {
-		copy[accHash] = make(map[common.Hash][]byte)
-		for slotHash, blob := range slots {
-			copy[accHash][slotHash] = blob
-		}
-	}
-	return copy
-}
-
-// TestMergeBasics tests some simple merges
-func TestMergeBasics(t *testing.T) {
-	var (
-		destructs = make(map[common.Hash]struct{})
-		accounts  = make(map[common.Hash][]byte)
-		storage   = make(map[common.Hash]map[common.Hash][]byte)
-	)
-	// Fill up a parent
-	for i := 0; i < 100; i++ {
-		h := randomHash()
-		data := randomAccount()
-
-		accounts[h] = data
-		if rand.Intn(4) == 0 {
-			destructs[h] = struct{}{}
-		}
-		if rand.Intn(2) == 0 {
-			accStorage := make(map[common.Hash][]byte)
-			value := make([]byte, 32)
-			rand.Read(value)
-			accStorage[randomHash()] = value
-			storage[h] = accStorage
-		}
-	}
-	// Add some (identical) layers on top
-	parent := newDiffLayer(emptyLayer(), common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage))
-	child := newDiffLayer(parent, common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage))
-	child = newDiffLayer(child, common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage))
-	child = newDiffLayer(child, common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage))
-	child = newDiffLayer(child, common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage))
-	// And flatten
-	merged := (child.flatten()).(*diffLayer)
-
-	{ // Check account lists
-		if have, want := len(merged.accountList), 0; have != want {
-			t.Errorf("accountList wrong: have %v, want %v", have, want)
-		}
-		if have, want := len(merged.AccountList()), len(accounts); have != want {
-			t.Errorf("AccountList() wrong: have %v, want %v", have, want)
-		}
-		if have, want := len(merged.accountList), len(accounts); have != want {
-			t.Errorf("accountList [2] wrong: have %v, want %v", have, want)
-		}
-	}
-	{ // Check account drops
-		if have, want := len(merged.destructSet), len(destructs); have != want {
-			t.Errorf("accountDrop wrong: have %v, want %v", have, want)
-		}
-	}
-	{ // Check storage lists
-		i := 0
-		for aHash, sMap := range storage {
-			if have, want := len(merged.storageList), i; have != want {
-				t.Errorf("[1] storageList wrong: have %v, want %v", have, want)
-			}
-			list, _ := merged.StorageList(aHash)
-			if have, want := len(list), len(sMap); have != want {
-				t.Errorf("[2] StorageList() wrong: have %v, want %v", have, want)
-			}
-			if have, want := len(merged.storageList[aHash]), len(sMap); have != want {
-				t.Errorf("storageList wrong: have %v, want %v", have, want)
-			}
-			i++
-		}
-	}
-}
-
-// TestMergeDelete tests some deletion
-func TestMergeDelete(t *testing.T) {
-	var (
-		storage = make(map[common.Hash]map[common.Hash][]byte)
-	)
-	// Fill up a parent
-	h1 := common.HexToHash("0x01")
-	h2 := common.HexToHash("0x02")
-
-	flipDrops := func() map[common.Hash]struct{} {
-		return map[common.Hash]struct{}{
-			h2: {},
-		}
-	}
-	flipAccs := func() map[common.Hash][]byte {
-		return map[common.Hash][]byte{
-			h1: randomAccount(),
-		}
-	}
-	flopDrops := func() map[common.Hash]struct{} {
-		return map[common.Hash]struct{}{
-			h1: {},
-		}
-	}
-	flopAccs := func() map[common.Hash][]byte {
-		return map[common.Hash][]byte{
-			h2: randomAccount(),
-		}
-	}
-	// Add some flipAccs-flopping layers on top
-	parent := newDiffLayer(emptyLayer(), common.Hash{}, flipDrops(), flipAccs(), storage)
-	child := parent.Update(common.Hash{}, flopDrops(), flopAccs(), storage)
-	child = child.Update(common.Hash{}, flipDrops(), flipAccs(), storage)
-	child = child.Update(common.Hash{}, flopDrops(), flopAccs(), storage)
-	child = child.Update(common.Hash{}, flipDrops(), flipAccs(), storage)
-	child = child.Update(common.Hash{}, flopDrops(), flopAccs(), storage)
-	child = child.Update(common.Hash{}, flipDrops(), flipAccs(), storage)
-
-	if data, _ := child.Account(h1); data == nil {
-		t.Errorf("last diff layer: expected %x account to be non-nil", h1)
-	}
-	if data, _ := child.Account(h2); data != nil {
-		t.Errorf("last diff layer: expected %x account to be nil", h2)
-	}
-	if _, ok := child.destructSet[h1]; ok {
-		t.Errorf("last diff layer: expected %x drop to be missing", h1)
-	}
-	if _, ok := child.destructSet[h2]; !ok {
-		t.Errorf("last diff layer: expected %x drop to be present", h1)
-	}
-	// And flatten
-	merged := (child.flatten()).(*diffLayer)
-
-	if data, _ := merged.Account(h1); data == nil {
-		t.Errorf("merged layer: expected %x account to be non-nil", h1)
-	}
-	if data, _ := merged.Account(h2); data != nil {
-		t.Errorf("merged layer: expected %x account to be nil", h2)
-	}
-	if _, ok := merged.destructSet[h1]; !ok { // Note, drops stay alive until persisted to disk!
-		t.Errorf("merged diff layer: expected %x drop to be present", h1)
-	}
-	if _, ok := merged.destructSet[h2]; !ok { // Note, drops stay alive until persisted to disk!
-		t.Errorf("merged diff layer: expected %x drop to be present", h1)
-	}
-	// If we add more granular metering of memory, we can enable this again,
-	// but it's not implemented for now
-	//if have, want := merged.memory, child.memory; have != want {
-	//	t.Errorf("mem wrong: have %d, want %d", have, want)
-	//}
-}
-
-// This tests that if we create a new account, and set a slot, and then merge
-// it, the lists will be correct.
-func TestInsertAndMerge(t *testing.T) {
-	// Fill up a parent
-	var (
-		acc    = common.HexToHash("0x01")
-		slot   = common.HexToHash("0x02")
-		parent *diffLayer
-		child  *diffLayer
-	)
-	{
-		var (
-			destructs = make(map[common.Hash]struct{})
-			accounts  = make(map[common.Hash][]byte)
-			storage   = make(map[common.Hash]map[common.Hash][]byte)
-		)
-		parent = newDiffLayer(emptyLayer(), common.Hash{}, destructs, accounts, storage)
-	}
-	{
-		var (
-			destructs = make(map[common.Hash]struct{})
-			accounts  = make(map[common.Hash][]byte)
-			storage   = make(map[common.Hash]map[common.Hash][]byte)
-		)
-		accounts[acc] = randomAccount()
-		storage[acc] = make(map[common.Hash][]byte)
-		storag