From 78745551c077bf54151202138c2629f288769561 Mon Sep 17 00:00:00 2001 From: Determinant Date: Tue, 15 Sep 2020 23:55:34 -0400 Subject: WIP: geth-tavum --- core/rawdb/accessors_indexes.go | 68 ++++++++++++++++--- core/rawdb/accessors_metadata.go | 25 ++----- core/rawdb/accessors_snapshot.go | 120 +++++++++++++++++++++++++++++++++ core/rawdb/database.go | 61 ++++++++++++----- core/rawdb/freezer.go | 141 ++++++++++++++++++++++++++++++--------- core/rawdb/freezer_reinit.go | 127 ----------------------------------- core/rawdb/freezer_table.go | 72 ++++++++++++-------- core/rawdb/schema.go | 56 ++++++++++++++-- core/rawdb/table.go | 93 ++++++++++++++++++++------ 9 files changed, 508 insertions(+), 255 deletions(-) create mode 100644 core/rawdb/accessors_snapshot.go delete mode 100644 core/rawdb/freezer_reinit.go (limited to 'core/rawdb') diff --git a/core/rawdb/accessors_indexes.go b/core/rawdb/accessors_indexes.go index 1dd478a..bf3ba07 100644 --- a/core/rawdb/accessors_indexes.go +++ b/core/rawdb/accessors_indexes.go @@ -17,14 +17,15 @@ package rawdb import ( + "bytes" "math/big" "github.com/ava-labs/coreth/core/types" "github.com/ava-labs/coreth/params" - "github.com/ava-labs/go-ethereum/common" - "github.com/ava-labs/go-ethereum/ethdb" - "github.com/ava-labs/go-ethereum/log" - "github.com/ava-labs/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" ) // ReadTxLookupEntry retrieves the positional metadata associated with a transaction @@ -52,20 +53,44 @@ func ReadTxLookupEntry(db ethdb.Reader, hash common.Hash) *uint64 { return &entry.BlockIndex } -// WriteTxLookupEntries stores a positional metadata for every transaction from +// writeTxLookupEntry stores a positional metadata for a transaction, +// enabling hash based transaction and receipt lookups. +func writeTxLookupEntry(db ethdb.KeyValueWriter, hash common.Hash, numberBytes []byte) { + if err := db.Put(txLookupKey(hash), numberBytes); err != nil { + log.Crit("Failed to store transaction lookup entry", "err", err) + } +} + +// WriteTxLookupEntries is identical to WriteTxLookupEntry, but it works on +// a list of hashes +func WriteTxLookupEntries(db ethdb.KeyValueWriter, number uint64, hashes []common.Hash) { + numberBytes := new(big.Int).SetUint64(number).Bytes() + for _, hash := range hashes { + writeTxLookupEntry(db, hash, numberBytes) + } +} + +// WriteTxLookupEntriesByBlock stores a positional metadata for every transaction from // a block, enabling hash based transaction and receipt lookups. -func WriteTxLookupEntries(db ethdb.KeyValueWriter, block *types.Block) { - number := block.Number().Bytes() +func WriteTxLookupEntriesByBlock(db ethdb.KeyValueWriter, block *types.Block) { + numberBytes := block.Number().Bytes() for _, tx := range block.Transactions() { - if err := db.Put(txLookupKey(tx.Hash()), number); err != nil { - log.Crit("Failed to store transaction lookup entry", "err", err) - } + writeTxLookupEntry(db, tx.Hash(), numberBytes) } } // DeleteTxLookupEntry removes all transaction data associated with a hash. func DeleteTxLookupEntry(db ethdb.KeyValueWriter, hash common.Hash) { - db.Delete(txLookupKey(hash)) + if err := db.Delete(txLookupKey(hash)); err != nil { + log.Crit("Failed to delete transaction lookup entry", "err", err) + } +} + +// DeleteTxLookupEntries removes all transaction lookups for a given block. +func DeleteTxLookupEntries(db ethdb.KeyValueWriter, hashes []common.Hash) { + for _, hash := range hashes { + DeleteTxLookupEntry(db, hash) + } } // ReadTransaction retrieves a specific transaction from the database, along with @@ -129,3 +154,24 @@ func WriteBloomBits(db ethdb.KeyValueWriter, bit uint, section uint64, head comm log.Crit("Failed to store bloom bits", "err", err) } } + +// DeleteBloombits removes all compressed bloom bits vector belonging to the +// given section range and bit index. +func DeleteBloombits(db ethdb.Database, bit uint, from uint64, to uint64) { + start, end := bloomBitsKey(bit, from, common.Hash{}), bloomBitsKey(bit, to, common.Hash{}) + it := db.NewIterator(nil, start) + defer it.Release() + + for it.Next() { + if bytes.Compare(it.Key(), end) >= 0 { + break + } + if len(it.Key()) != len(bloomBitsPrefix)+2+8+32 { + continue + } + db.Delete(it.Key()) + } + if it.Error() != nil { + log.Crit("Failed to delete bloom bits", "err", it.Error()) + } +} diff --git a/core/rawdb/accessors_metadata.go b/core/rawdb/accessors_metadata.go index 7a17123..c5e5655 100644 --- a/core/rawdb/accessors_metadata.go +++ b/core/rawdb/accessors_metadata.go @@ -20,10 +20,10 @@ import ( "encoding/json" "github.com/ava-labs/coreth/params" - "github.com/ava-labs/go-ethereum/common" - "github.com/ava-labs/go-ethereum/ethdb" - "github.com/ava-labs/go-ethereum/log" - "github.com/ava-labs/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" ) // ReadDatabaseVersion retrieves the version number of the database. @@ -79,20 +79,3 @@ func WriteChainConfig(db ethdb.KeyValueWriter, hash common.Hash, cfg *params.Cha log.Crit("Failed to store chain config", "err", err) } } - -// ReadPreimage retrieves a single preimage of the provided hash. -func ReadPreimage(db ethdb.KeyValueReader, hash common.Hash) []byte { - data, _ := db.Get(preimageKey(hash)) - return data -} - -// WritePreimages writes the provided set of preimages to the database. -func WritePreimages(db ethdb.KeyValueWriter, preimages map[common.Hash][]byte) { - for hash, preimage := range preimages { - if err := db.Put(preimageKey(hash), preimage); err != nil { - log.Crit("Failed to store trie preimage", "err", err) - } - } - preimageCounter.Inc(int64(len(preimages))) - preimageHitCounter.Inc(int64(len(preimages))) -} diff --git a/core/rawdb/accessors_snapshot.go b/core/rawdb/accessors_snapshot.go new file mode 100644 index 0000000..ecd4e65 --- /dev/null +++ b/core/rawdb/accessors_snapshot.go @@ -0,0 +1,120 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rawdb + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" +) + +// ReadSnapshotRoot retrieves the root of the block whose state is contained in +// the persisted snapshot. +func ReadSnapshotRoot(db ethdb.KeyValueReader) common.Hash { + data, _ := db.Get(snapshotRootKey) + if len(data) != common.HashLength { + return common.Hash{} + } + return common.BytesToHash(data) +} + +// WriteSnapshotRoot stores the root of the block whose state is contained in +// the persisted snapshot. +func WriteSnapshotRoot(db ethdb.KeyValueWriter, root common.Hash) { + if err := db.Put(snapshotRootKey, root[:]); err != nil { + log.Crit("Failed to store snapshot root", "err", err) + } +} + +// DeleteSnapshotRoot deletes the hash of the block whose state is contained in +// the persisted snapshot. Since snapshots are not immutable, this method can +// be used during updates, so a crash or failure will mark the entire snapshot +// invalid. +func DeleteSnapshotRoot(db ethdb.KeyValueWriter) { + if err := db.Delete(snapshotRootKey); err != nil { + log.Crit("Failed to remove snapshot root", "err", err) + } +} + +// ReadAccountSnapshot retrieves the snapshot entry of an account trie leaf. +func ReadAccountSnapshot(db ethdb.KeyValueReader, hash common.Hash) []byte { + data, _ := db.Get(accountSnapshotKey(hash)) + return data +} + +// WriteAccountSnapshot stores the snapshot entry of an account trie leaf. +func WriteAccountSnapshot(db ethdb.KeyValueWriter, hash common.Hash, entry []byte) { + if err := db.Put(accountSnapshotKey(hash), entry); err != nil { + log.Crit("Failed to store account snapshot", "err", err) + } +} + +// DeleteAccountSnapshot removes the snapshot entry of an account trie leaf. +func DeleteAccountSnapshot(db ethdb.KeyValueWriter, hash common.Hash) { + if err := db.Delete(accountSnapshotKey(hash)); err != nil { + log.Crit("Failed to delete account snapshot", "err", err) + } +} + +// ReadStorageSnapshot retrieves the snapshot entry of an storage trie leaf. +func ReadStorageSnapshot(db ethdb.KeyValueReader, accountHash, storageHash common.Hash) []byte { + data, _ := db.Get(storageSnapshotKey(accountHash, storageHash)) + return data +} + +// WriteStorageSnapshot stores the snapshot entry of an storage trie leaf. +func WriteStorageSnapshot(db ethdb.KeyValueWriter, accountHash, storageHash common.Hash, entry []byte) { + if err := db.Put(storageSnapshotKey(accountHash, storageHash), entry); err != nil { + log.Crit("Failed to store storage snapshot", "err", err) + } +} + +// DeleteStorageSnapshot removes the snapshot entry of an storage trie leaf. +func DeleteStorageSnapshot(db ethdb.KeyValueWriter, accountHash, storageHash common.Hash) { + if err := db.Delete(storageSnapshotKey(accountHash, storageHash)); err != nil { + log.Crit("Failed to delete storage snapshot", "err", err) + } +} + +// IterateStorageSnapshots returns an iterator for walking the entire storage +// space of a specific account. +func IterateStorageSnapshots(db ethdb.Iteratee, accountHash common.Hash) ethdb.Iterator { + return db.NewIterator(storageSnapshotsKey(accountHash), nil) +} + +// ReadSnapshotJournal retrieves the serialized in-memory diff layers saved at +// the last shutdown. The blob is expected to be max a few 10s of megabytes. +func ReadSnapshotJournal(db ethdb.KeyValueReader) []byte { + data, _ := db.Get(snapshotJournalKey) + return data +} + +// WriteSnapshotJournal stores the serialized in-memory diff layers to save at +// shutdown. The blob is expected to be max a few 10s of megabytes. +func WriteSnapshotJournal(db ethdb.KeyValueWriter, journal []byte) { + if err := db.Put(snapshotJournalKey, journal); err != nil { + log.Crit("Failed to store snapshot journal", "err", err) + } +} + +// DeleteSnapshotJournal deletes the serialized in-memory diff layers saved at +// the last shutdown +func DeleteSnapshotJournal(db ethdb.KeyValueWriter) { + if err := db.Delete(snapshotJournalKey); err != nil { + log.Crit("Failed to remove snapshot journal", "err", err) + } +} diff --git a/core/rawdb/database.go b/core/rawdb/database.go index f04c34f..316b5ad 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -21,13 +21,14 @@ import ( "errors" "fmt" "os" + "sync/atomic" "time" - "github.com/ava-labs/go-ethereum/common" - "github.com/ava-labs/go-ethereum/ethdb" - "github.com/ava-labs/go-ethereum/ethdb/leveldb" - "github.com/ava-labs/go-ethereum/ethdb/memorydb" - "github.com/ava-labs/go-ethereum/log" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/ethdb/leveldb" + "github.com/ethereum/go-ethereum/ethdb/memorydb" + "github.com/ethereum/go-ethereum/log" "github.com/olekukonko/tablewriter" ) @@ -41,10 +42,10 @@ type freezerdb struct { // the slow ancient tables. func (frdb *freezerdb) Close() error { var errs []error - if err := frdb.KeyValueStore.Close(); err != nil { + if err := frdb.AncientStore.Close(); err != nil { errs = append(errs, err) } - if err := frdb.AncientStore.Close(); err != nil { + if err := frdb.KeyValueStore.Close(); err != nil { errs = append(errs, err) } if len(errs) != 0 { @@ -53,6 +54,22 @@ func (frdb *freezerdb) Close() error { return nil } +// Freeze is a helper method used for external testing to trigger and block until +// a freeze cycle completes, without having to sleep for a minute to trigger the +// automatic background run. +func (frdb *freezerdb) Freeze(threshold uint64) { + // Set the freezer threshold to a temporary value + defer func(old uint64) { + atomic.StoreUint64(&frdb.AncientStore.(*freezer).threshold, old) + }(atomic.LoadUint64(&frdb.AncientStore.(*freezer).threshold)) + atomic.StoreUint64(&frdb.AncientStore.(*freezer).threshold, threshold) + + // Trigger a freeze cycle and block until it's done + trigger := make(chan struct{}, 1) + frdb.AncientStore.(*freezer).trigger <- trigger + <-trigger +} + // nofreezedb is a database wrapper that disables freezer data retrievals. type nofreezedb struct { ethdb.KeyValueStore @@ -137,7 +154,10 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace st // If the freezer already contains something, ensure that the genesis blocks // match, otherwise we might mix up freezers across chains and destroy both // the freezer and the key-value store. - if frgenesis, _ := frdb.Ancient(freezerHashTable, 0); !bytes.Equal(kvgenesis, frgenesis) { + frgenesis, err := frdb.Ancient(freezerHashTable, 0) + if err != nil { + return nil, fmt.Errorf("failed to retrieve genesis from ancient %v", err) + } else if !bytes.Equal(kvgenesis, frgenesis) { return nil, fmt.Errorf("genesis mismatch: %#x (leveldb) != %#x (ancients)", kvgenesis, frgenesis) } // Key-value store and freezer belong to the same network. Ensure that they @@ -150,11 +170,10 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace st } // Database contains only older data than the freezer, this happens if the // state was wiped and reinited from an existing freezer. - } else { - // Key-value store continues where the freezer left off, all is fine. We might - // have duplicate blocks (crash after freezer write but before kay-value store - // deletion, but that's fine). } + // Otherwise, key-value store continues where the freezer left off, all is fine. + // We might have duplicate blocks (crash after freezer write but before key-value + // store deletion, but that's fine). } else { // If the freezer is empty, ensure nothing was moved yet from the key-value // store, otherwise we'll end up missing data. We check block #1 to decide @@ -167,9 +186,9 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace st return nil, errors.New("ancient chain segments already extracted, please set --datadir.ancient to the correct path") } // Block #1 is still in the database, we're allowed to init a new feezer - } else { - // The head header is still the genesis, we're allowed to init a new feezer } + // Otherwise, the head header is still the genesis, we're allowed to init a new + // feezer. } } // Freezer is consistent with the key-value database, permit combining the two @@ -222,7 +241,7 @@ func NewLevelDBDatabaseWithFreezer(file string, cache int, handles int, freezer // InspectDatabase traverses the entire database and checks the size // of all different categories of data. func InspectDatabase(db ethdb.Database) error { - it := db.NewIterator() + it := db.NewIterator(nil, nil) defer it.Release() var ( @@ -239,7 +258,10 @@ func InspectDatabase(db ethdb.Database) error { numHashPairing common.StorageSize hashNumPairing common.StorageSize trieSize common.StorageSize + codeSize common.StorageSize txlookupSize common.StorageSize + accountSnapSize common.StorageSize + storageSnapSize common.StorageSize preimageSize common.StorageSize bloomBitsSize common.StorageSize cliqueSnapsSize common.StorageSize @@ -281,6 +303,10 @@ func InspectDatabase(db ethdb.Database) error { receiptSize += size case bytes.HasPrefix(key, txLookupPrefix) && len(key) == (len(txLookupPrefix)+common.HashLength): txlookupSize += size + case bytes.HasPrefix(key, SnapshotAccountPrefix) && len(key) == (len(SnapshotAccountPrefix)+common.HashLength): + accountSnapSize += size + case bytes.HasPrefix(key, SnapshotStoragePrefix) && len(key) == (len(SnapshotStoragePrefix)+2*common.HashLength): + storageSnapSize += size case bytes.HasPrefix(key, preimagePrefix) && len(key) == (len(preimagePrefix)+common.HashLength): preimageSize += size case bytes.HasPrefix(key, bloomBitsPrefix) && len(key) == (len(bloomBitsPrefix)+10+common.HashLength): @@ -291,6 +317,8 @@ func InspectDatabase(db ethdb.Database) error { chtTrieNodes += size case bytes.HasPrefix(key, []byte("blt-")) && len(key) == 4+common.HashLength: bloomTrieNodes += size + case bytes.HasPrefix(key, codePrefix) && len(key) == len(codePrefix)+common.HashLength: + codeSize += size case len(key) == common.HashLength: trieSize += size default: @@ -330,8 +358,11 @@ func InspectDatabase(db ethdb.Database) error { {"Key-Value store", "Block hash->number", hashNumPairing.String()}, {"Key-Value store", "Transaction index", txlookupSize.String()}, {"Key-Value store", "Bloombit index", bloomBitsSize.String()}, + {"Key-Value store", "Contract codes", codeSize.String()}, {"Key-Value store", "Trie nodes", trieSize.String()}, {"Key-Value store", "Trie preimages", preimageSize.String()}, + {"Key-Value store", "Account snapshot", accountSnapSize.String()}, + {"Key-Value store", "Storage snapshot", storageSnapSize.String()}, {"Key-Value store", "Clique snapshots", cliqueSnapsSize.String()}, {"Key-Value store", "Singleton metadata", metadata.String()}, {"Ancient store", "Headers", ancientHeaders.String()}, diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index ce2e879..433290b 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -22,14 +22,15 @@ import ( "math" "os" "path/filepath" + "sync" "sync/atomic" "time" "github.com/ava-labs/coreth/params" - "github.com/ava-labs/go-ethereum/common" - "github.com/ava-labs/go-ethereum/ethdb" - "github.com/ava-labs/go-ethereum/log" - "github.com/ava-labs/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/prometheus/tsdb/fileutil" ) @@ -69,10 +70,16 @@ type freezer struct { // WARNING: The `frozen` field is accessed atomically. On 32 bit platforms, only // 64-bit aligned fields can be atomic. The struct is guaranteed to be so aligned, // so take advantage of that (https://golang.org/pkg/sync/atomic/#pkg-note-BUG). - frozen uint64 // Number of blocks already frozen + frozen uint64 // Number of blocks already frozen + threshold uint64 // Number of recent blocks not to freeze (params.FullImmutabilityThreshold apart from tests) tables map[string]*freezerTable // Data tables for storing everything instanceLock fileutil.Releaser // File-system lock to prevent double opens + + trigger chan chan struct{} // Manual blocking freeze trigger, test determinism + + quit chan struct{} + closeOnce sync.Once } // newFreezer creates a chain freezer that moves ancient chain data into @@ -80,9 +87,9 @@ type freezer struct { func newFreezer(datadir string, namespace string) (*freezer, error) { // Create the initial freezer object var ( - readMeter = metrics.NewRegisteredMeter(namespace+"ancient/read", nil) - writeMeter = metrics.NewRegisteredMeter(namespace+"ancient/write", nil) - sizeCounter = metrics.NewRegisteredCounter(namespace+"ancient/size", nil) + readMeter = metrics.NewRegisteredMeter(namespace+"ancient/read", nil) + writeMeter = metrics.NewRegisteredMeter(namespace+"ancient/write", nil) + sizeGauge = metrics.NewRegisteredGauge(namespace+"ancient/size", nil) ) // Ensure the datadir is not a symbolic link if it exists. if info, err := os.Lstat(datadir); !os.IsNotExist(err) { @@ -99,11 +106,14 @@ func newFreezer(datadir string, namespace string) (*freezer, error) { } // Open all the supported data tables freezer := &freezer{ + threshold: params.FullImmutabilityThreshold, tables: make(map[string]*freezerTable), instanceLock: lock, + trigger: make(chan chan struct{}), + quit: make(chan struct{}), } for name, disableSnappy := range freezerNoSnappy { - table, err := newTable(datadir, name, readMeter, writeMeter, sizeCounter, disableSnappy) + table, err := newTable(datadir, name, readMeter, writeMeter, sizeGauge, disableSnappy) if err != nil { for _, table := range freezer.tables { table.Close() @@ -127,14 +137,17 @@ func newFreezer(datadir string, namespace string) (*freezer, error) { // Close terminates the chain freezer, unmapping all the data files. func (f *freezer) Close() error { var errs []error - for _, table := range f.tables { - if err := table.Close(); err != nil { + f.closeOnce.Do(func() { + f.quit <- struct{}{} + for _, table := range f.tables { + if err := table.Close(); err != nil { + errs = append(errs, err) + } + } + if err := f.instanceLock.Release(); err != nil { errs = append(errs, err) } - } - if err := f.instanceLock.Release(); err != nil { - errs = append(errs, err) - } + }) if errs != nil { return fmt.Errorf("%v", errs) } @@ -218,7 +231,7 @@ func (f *freezer) AppendAncient(number uint64, hash, header, body, receipts, td return nil } -// Truncate discards any recent data above the provided threshold number. +// TruncateAncients discards any recent data above the provided threshold number. func (f *freezer) TruncateAncients(items uint64) error { if atomic.LoadUint64(&f.frozen) <= items { return nil @@ -232,7 +245,7 @@ func (f *freezer) TruncateAncients(items uint64) error { return nil } -// sync flushes all data tables to disk. +// Sync flushes all data tables to disk. func (f *freezer) Sync() error { var errs []error for _, table := range f.tables { @@ -254,48 +267,75 @@ func (f *freezer) Sync() error { func (f *freezer) freeze(db ethdb.KeyValueStore) { nfdb := &nofreezedb{KeyValueStore: db} + var ( + backoff bool + triggered chan struct{} // Used in tests + ) for { + select { + case <-f.quit: + log.Info("Freezer shutting down") + return + default: + } + if backoff { + // If we were doing a manual trigger, notify it + if triggered != nil { + triggered <- struct{}{} + triggered = nil + } + select { + case <-time.NewTimer(freezerRecheckInterval).C: + backoff = false + case triggered = <-f.trigger: + backoff = false + case <-f.quit: + return + } + } // Retrieve the freezing threshold. hash := ReadHeadBlockHash(nfdb) if hash == (common.Hash{}) { log.Debug("Current full block hash unavailable") // new chain, empty database - time.Sleep(freezerRecheckInterval) + backoff = true continue } number := ReadHeaderNumber(nfdb, hash) + threshold := atomic.LoadUint64(&f.threshold) + switch { case number == nil: log.Error("Current full block number unavailable", "hash", hash) - time.Sleep(freezerRecheckInterval) + backoff = true continue - case *number < params.ImmutabilityThreshold: - log.Debug("Current full block not old enough", "number", *number, "hash", hash, "delay", params.ImmutabilityThreshold) - time.Sleep(freezerRecheckInterval) + case *number < threshold: + log.Debug("Current full block not old enough", "number", *number, "hash", hash, "delay", threshold) + backoff = true continue - case *number-params.ImmutabilityThreshold <= f.frozen: + case *number-threshold <= f.frozen: log.Debug("Ancient blocks frozen already", "number", *number, "hash", hash, "frozen", f.frozen) - time.Sleep(freezerRecheckInterval) + backoff = true continue } head := ReadHeader(nfdb, hash, *number) if head == nil { log.Error("Current full block unavailable", "number", *number, "hash", hash) - time.Sleep(freezerRecheckInterval) + backoff = true continue } // Seems we have data ready to be frozen, process in usable batches - limit := *number - params.ImmutabilityThreshold + limit := *number - threshold if limit-f.frozen > freezerBatchLimit { limit = f.frozen + freezerBatchLimit } var ( start = time.Now() first = f.frozen - ancients = make([]common.Hash, 0, limit) + ancients = make([]common.Hash, 0, limit-f.frozen) ) - for f.frozen < limit { + for f.frozen <= limit { // Retrieves all the components of the canonical block hash := ReadCanonicalHash(nfdb, f.frozen) if hash == (common.Hash{}) { @@ -346,11 +386,15 @@ func (f *freezer) freeze(db ethdb.KeyValueStore) { log.Crit("Failed to delete frozen canonical blocks", "err", err) } batch.Reset() - // Wipe out side chain also. + + // Wipe out side chains also and track dangling side chians + var dangling []common.Hash for number := first; number < f.frozen; number++ { // Always keep the genesis block in active database if number != 0 { - for _, hash := range ReadAllHashes(db, number) { + dangling = ReadAllHashes(db, number) + for _, hash := range dangling { + log.Trace("Deleting side chain", "number", number, "hash", hash) DeleteBlock(batch, hash, number) } } @@ -358,6 +402,41 @@ func (f *freezer) freeze(db ethdb.KeyValueStore) { if err := batch.Write(); err != nil { log.Crit("Failed to delete frozen side blocks", "err", err) } + batch.Reset() + + // Step into the future and delete and dangling side chains + if f.frozen > 0 { + tip := f.frozen + for len(dangling) > 0 { + drop := make(map[common.Hash]struct{}) + for _, hash := range dangling { + log.Debug("Dangling parent from freezer", "number", tip-1, "hash", hash) + drop[hash] = struct{}{} + } + children := ReadAllHashes(db, tip) + for i := 0; i < len(children); i++ { + // Dig up the child and ensure it's dangling + child := ReadHeader(nfdb, children[i], tip) + if child == nil { + log.Error("Missing dangling header", "number", tip, "hash", children[i]) + continue + } + if _, ok := drop[child.ParentHash]; !ok { + children = append(children[:i], children[i+1:]...) + i-- + continue + } + // Delete all block data associated with the child + log.Debug("Deleting dangling block", "number", tip, "hash", children[i], "parent", child.ParentHash) + DeleteBlock(batch, children[i], tip) + } + dangling = children + tip++ + } + if err := batch.Write(); err != nil { + log.Crit("Failed to delete dangling side blocks", "err", err) + } + } // Log something friendly for the user context := []interface{}{ "blocks", f.frozen - first, "elapsed", common.PrettyDuration(time.Since(start)), "number", f.frozen - 1, @@ -369,7 +448,7 @@ func (f *freezer) freeze(db ethdb.KeyValueStore) { // Avoid database thrashing with tiny writes if f.frozen-first < freezerBatchLimit { - time.Sleep(freezerRecheckInterval) + backoff = true } } } diff --git a/core/rawdb/freezer_reinit.go b/core/rawdb/freezer_reinit.go deleted file mode 100644 index 6b9fb79..0000000 --- a/core/rawdb/freezer_reinit.go +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright 2019 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package rawdb - -import ( - "errors" - "runtime" - "sync/atomic" - "time" - - "github.com/ava-labs/coreth/core/types" - "github.com/ava-labs/go-ethereum/common" - "github.com/ava-labs/go-ethereum/common/prque" - "github.com/ava-labs/go-ethereum/ethdb" - "github.com/ava-labs/go-ethereum/log" -) - -// InitDatabaseFromFreezer reinitializes an empty database from a previous batch -// of frozen ancient blocks. The method iterates over all the frozen blocks and -// injects into the database the block hash->number mappings and the transaction -// lookup entries. -func InitDatabaseFromFreezer(db ethdb.Database) error { - // If we can't access the freezer or it's empty, abort - frozen, err := db.Ancients() - if err != nil || frozen == 0 { - return err - } - // Blocks previously frozen, iterate over- and hash them concurrently - var ( - number = ^uint64(0) // -1 - results = make(chan *types.Block, 4*runtime.NumCPU()) - ) - abort := make(chan struct{}) - defer close(abort) - - for i := 0; i < runtime.NumCPU(); i++ { - go func() { - for { - // Fetch the next task number, terminating if everything's done - n := atomic.AddUint64(&number, 1) - if n >= frozen { - return - } - // Retrieve the block from the freezer (no need for the hash, we pull by - // number from the freezer). If successful, pre-cache the block hash and - // the individual transaction hashes for storing into the database. - block := ReadBlock(db, common.Hash{}, n) - if block != nil { - block.Hash() - for _, tx := range block.Transactions() { - tx.Hash() - } - } - // Feed the block to the aggregator, or abort on interrupt - select { - case results <- block: - case <-abort: - return - } - } - }() - } - // Reassemble the blocks into a contiguous stream and push them out to disk - var ( - queue = prque.New(nil) - next = int64(0) - - batch = db.NewBatch() - start = time.Now() - logged time.Time - ) - for i := uint64(0); i < frozen; i++ { - // Retrieve the next result and bail if it's nil - block := <-results - if block == nil { - return errors.New("broken ancient database") - } - // Push the block into the import queue and process contiguous ranges - queue.Push(block, -int64(block.NumberU64())) - for !queue.Empty() { - // If the next available item is gapped, return - if _, priority := queue.Peek(); -priority != next { - break - } - // Next block available, pop it off and index it - block = queue.PopItem().(*types.Block) - next++ - - // Inject hash<->number mapping and txlookup indexes - WriteHeaderNumber(batch, block.Hash(), block.NumberU64()) - WriteTxLookupEntries(batch, block) - - // If enough data was accumulated in memory or we're at the last block, dump to disk - if batch.ValueSize() > ethdb.IdealBatchSize || uint64(next) == frozen { - if err := batch.Write(); err != nil { - return err - } - batch.Reset() - } - // If we've spent too much time already, notify the user of what we're doing - if time.Since(logged) > 8*time.Second { - log.Info("Initializing chain from ancient data", "number", block.Number(), "hash", block.Hash(), "total", frozen-1, "elapsed", common.PrettyDuration(time.Since(start))) - logged = time.Now() - } - } - } - hash := ReadCanonicalHash(db, frozen-1) - WriteHeadHeaderHash(db, hash) - WriteHeadFastBlockHash(db, hash) - - log.Info("Initialized chain from ancient data", "number", frozen-1, "hash", hash, "elapsed", common.PrettyDuration(time.Since(start))) - return nil -} diff --git a/core/rawdb/freezer_table.go b/core/rawdb/freezer_table.go index fc72669..b9d8a27 100644 --- a/core/rawdb/freezer_table.go +++ b/core/rawdb/freezer_table.go @@ -26,9 +26,9 @@ import ( "sync" "sync/atomic" - "github.com/ava-labs/go-ethereum/common" - "github.com/ava-labs/go-ethereum/log" - "github.com/ava-labs/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/golang/snappy" ) @@ -94,18 +94,18 @@ type freezerTable struct { // to count how many historic items have gone missing. itemOffset uint32 // Offset (number of discarded items) - headBytes uint32 // Number of bytes written to the head file - readMeter metrics.Meter // Meter for measuring the effective amount of data read - writeMeter metrics.Meter // Meter for measuring the effective amount of data written - sizeCounter metrics.Counter // Counter for tracking the combined size of all freezer tables + headBytes uint32 // Number of bytes written to the head file + readMeter metrics.Meter // Meter for measuring the effective amount of data read + writeMeter metrics.Meter // Meter for measuring the effective amount of data written + sizeGauge metrics.Gauge // Gauge for tracking the combined size of all freezer tables logger log.Logger // Logger with database path and table name ambedded lock sync.RWMutex // Mutex protecting the data file descriptors } // newTable opens a freezer table with default settings - 2G files -func newTable(path string, name string, readMeter metrics.Meter, writeMeter metrics.Meter, sizeCounter metrics.Counter, disableSnappy bool) (*freezerTable, error) { - return newCustomTable(path, name, readMeter, writeMeter, sizeCounter, 2*1000*1000*1000, disableSnappy) +func newTable(path string, name string, readMeter metrics.Meter, writeMeter metrics.Meter, sizeGauge metrics.Gauge, disableSnappy bool) (*freezerTable, error) { + return newCustomTable(path, name, readMeter, writeMeter, sizeGauge, 2*1000*1000*1000, disableSnappy) } // openFreezerFileForAppend opens a freezer table file and seeks to the end @@ -149,7 +149,7 @@ func truncateFreezerFile(file *os.File, size int64) error { // newCustomTable opens a freezer table, creating the data and index files if they are // non existent. Both files are truncated to the shortest common length to ensure // they don't go out of sync. -func newCustomTable(path string, name string, readMeter metrics.Meter, writeMeter metrics.Meter, sizeCounter metrics.Counter, maxFilesize uint32, noCompression bool) (*freezerTable, error) { +func newCustomTable(path string, name string, readMeter metrics.Meter, writeMeter metrics.Meter, sizeGauge metrics.Gauge, maxFilesize uint32, noCompression bool) (*freezerTable, error) { // Ensure the containing directory exists and open the indexEntry file if err := os.MkdirAll(path, 0755); err != nil { return nil, err @@ -172,7 +172,7 @@ func newCustomTable(path string, name string, readMeter metrics.Meter, writeMete files: make(map[uint32]*os.File), readMeter: readMeter, writeMeter: writeMeter, - sizeCounter: sizeCounter, + sizeGauge: sizeGauge, name: name, path: path, logger: log.New("database", path, "table", name), @@ -189,7 +189,7 @@ func newCustomTable(path string, name string, readMeter metrics.Meter, writeMete tab.Close() return nil, err } - tab.sizeCounter.Inc(int64(size)) + tab.sizeGauge.Inc(int64(size)) return tab, nil } @@ -232,8 +232,8 @@ func (t *freezerTable) repair() error { t.index.ReadAt(buffer, 0) firstIndex.unmarshalBinary(buffer) - t.tailId = firstIndex.offset - t.itemOffset = firstIndex.filenum + t.tailId = firstIndex.filenum + t.itemOffset = firstIndex.offset t.index.ReadAt(buffer, offsetsSize-indexEntrySize) lastIndex.unmarshalBinary(buffer) @@ -330,7 +330,8 @@ func (t *freezerTable) truncate(items uint64) error { defer t.lock.Unlock() // If our item count is correct, don't do anything - if atomic.LoadUint64(&t.items) <= items { + existing := atomic.LoadUint64(&t.items) + if existing <= items { return nil } // We need to truncate, save the old size for metrics tracking @@ -339,7 +340,11 @@ func (t *freezerTable) truncate(items uint64) error { return err } // Something's out of sync, truncate the table's offset index - t.logger.Warn("Truncating freezer table", "items", t.items, "limit", items) + log := t.logger.Debug + if existing > items+1 { + log = t.logger.Warn // Only loud warn if we delete multiple items + } + log("Truncating freezer table", "items", existing, "limit", items) if err := truncateFreezerFile(t.index, int64(items+1)*indexEntrySize); err != nil { return err } @@ -378,7 +383,7 @@ func (t *freezerTable) truncate(items uint64) error { if err != nil { return err } - t.sizeCounter.Dec(int64(oldSize - newSize)) + t.sizeGauge.Dec(int64(oldSize - newSize)) return nil } @@ -510,7 +515,7 @@ func (t *freezerTable) Append(item uint64, blob []byte) error { t.index.Write(idx.marshallBinary()) t.writeMeter.Mark(int64(bLen + indexEntrySize)) - t.sizeCounter.Inc(int64(bLen + indexEntrySize)) + t.sizeGauge.Inc(int64(bLen + indexEntrySize)) atomic.AddUint64(&t.items, 1) return nil @@ -519,16 +524,27 @@ func (t *freezerTable) Append(item uint64, blob []byte) error { // getBounds returns the indexes for the item // returns start, end, filenumber and error func (t *freezerTable) getBounds(item uint64) (uint32, uint32, uint32, error) { - var startIdx, endIdx indexEntry buffer := make([]byte, indexEntrySize) - if _, err := t.index.ReadAt(buffer, int64(item*indexEntrySize)); err != nil { - return 0, 0, 0, err - } - startIdx.unmarshalBinary(buffer) + var startIdx, endIdx indexEntry + // Read second index if _, err := t.index.ReadAt(buffer, int64((item+1)*indexEntrySize)); err != nil { return 0, 0, 0, err } endIdx.unmarshalBinary(buffer) + // Read first index (unless it's the very first item) + if item != 0 { + if _, err := t.index.ReadAt(buffer, int64(item*indexEntrySize)); err != nil { + return 0, 0, 0, err + } + startIdx.unmarshalBinary(buffer) + } else { + // Special case if we're reading the first item in the freezer. We assume that + // the first item always start from zero(regarding the deletion, we + // only support deletion by files, so that the assumption is held). + // This means we can use the first item metadata to carry information about + // the 'global' offset, for the deletion-case + return 0, endIdx.offset, endIdx.filenum, nil + } if startIdx.filenum != endIdx.filenum { // If a piece of data 'crosses' a data-file, // it's actually in one piece on the second data-file. @@ -541,20 +557,22 @@ func (t *freezerTable) getBounds(item uint64) (uint32, uint32, uint32, error) { // Retrieve looks up the data offset of an item with the given number and retrieves // the raw binary blob from the data file. func (t *freezerTable) Retrieve(item uint64) ([]byte, error) { + t.lock.RLock() // Ensure the table and the item is accessible if t.index == nil || t.head == nil { + t.lock.RUnlock() return nil, errClosed } if atomic.LoadUint64(&t.items) <= item { + t.lock.RUnlock() return nil, errOutOfBounds } // Ensure the item was not deleted from the tail either - offset := atomic.LoadUint32(&t.itemOffset) - if uint64(offset) > item { + if uint64(t.itemOffset) > item { + t.lock.RUnlock() return nil, errOutOfBounds } - t.lock.RLock() - startOffset, endOffset, filenum, err := t.getBounds(item - uint64(offset)) + startOffset, endOffset, filenum, err := t.getBounds(item - uint64(t.itemOffset)) if err != nil { t.lock.RUnlock() return nil, err diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 51d1f66..e2b093a 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -18,10 +18,11 @@ package rawdb import ( + "bytes" "encoding/binary" - "github.com/ava-labs/go-ethereum/common" - "github.com/ava-labs/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/metrics" ) // The fields below define the low level database schema prefixing. @@ -38,9 +39,24 @@ var ( // headFastBlockKey tracks the latest known incomplete block's hash during fast sync. headFastBlockKey = []byte("LastFast") + // lastPivotKey tracks the last pivot block used by fast sync (to reenable on sethead). + lastPivotKey = []byte("LastPivot") + // fastTrieProgressKey tracks the number of trie entries imported during fast sync. fastTrieProgressKey = []byte("TrieSync") + // snapshotRootKey tracks the hash of the last snapshot. + snapshotRootKey = []byte("SnapshotRoot") + + // snapshotJournalKey tracks the in-memory diff layers across restarts. + snapshotJournalKey = []byte("SnapshotJournal") + + // txIndexTailKey tracks the oldest block whose transactions have been indexed. + txIndexTailKey = []byte("TransactionIndexTail") + + // fastTxLookupLimitKey tracks the transaction lookup limit during fast sync. + fastTxLookupLimitKey = []byte("FastTransactionLookupLimit") + // Data item prefixes (use single byte to avoid mixing data types, avoid `i`, used for indexes). headerPrefix = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header headerTDSuffix = []byte("t") // headerPrefix + num (uint64 big endian) + hash + headerTDSuffix -> td @@ -50,8 +66,11 @@ var ( blockBodyPrefix = []byte("b") // blockBodyPrefix + num (uint64 big endian) + hash -> block body blockReceiptsPrefix = []byte("r") // blockReceiptsPrefix + num (uint64 big endian) + hash -> block receipts - txLookupPrefix = []byte("l") // txLookupPrefix + hash -> transaction/receipt lookup metadata - bloomBitsPrefix = []byte("B") // bloomBitsPrefix + bit (uint16 big endian) + section (uint64 big endian) + hash -> bloom bits + txLookupPrefix = []byte("l") // txLookupPrefix + hash -> transaction/receipt lookup metadata + bloomBitsPrefix = []byte("B") // bloomBitsPrefix + bit (uint16 big endian) + section (uint64 big endian) + hash -> bloom bits + SnapshotAccountPrefix = []byte("a") // SnapshotAccountPrefix + account hash -> account trie value + SnapshotStoragePrefix = []byte("o") // SnapshotStoragePrefix + account hash + storage hash -> storage trie value + codePrefix = []byte("c") // codePrefix + code hash -> account code preimagePrefix = []byte("secure-key-") // preimagePrefix + hash -> preimage configPrefix = []byte("ethereum-config-") // config prefix for the db @@ -145,6 +164,21 @@ func txLookupKey(hash common.Hash) []byte { return append(txLookupPrefix, hash.Bytes()...) } +// accountSnapshotKey = SnapshotAccountPrefix + hash +func accountSnapshotKey(hash common.Hash) []byte { + return append(SnapshotAccountPrefix, hash.Bytes()...) +} + +// storageSnapshotKey = SnapshotStoragePrefix + account hash + storage hash +func storageSnapshotKey(accountHash, storageHash common.Hash) []byte { + return append(append(SnapshotStoragePrefix, accountHash.Bytes()...), storageHash.Bytes()...) +} + +// storageSnapshotsKey = SnapshotStoragePrefix + account hash + storage hash +func storageSnapshotsKey(accountHash common.Hash) []byte { + return append(SnapshotStoragePrefix, accountHash.Bytes()...) +} + // bloomBitsKey = bloomBitsPrefix + bit (uint16 big endian) + section (uint64 big endian) + hash func bloomBitsKey(bit uint, section uint64, hash common.Hash) []byte { key := append(append(bloomBitsPrefix, make([]byte, 10)...), hash.Bytes()...) @@ -160,6 +194,20 @@ func preimageKey(hash common.Hash) []byte { return append(preimagePrefix, hash.Bytes()...) } +// codeKey = codePrefix + hash +func codeKey(hash common.Hash) []byte { + return append(codePrefix, hash.Bytes()...) +} + +// IsCodeKey reports whether the given byte slice is the key of contract code, +// if so return the raw code hash as well. +func IsCodeKey(key []byte) (bool, []byte) { + if bytes.HasPrefix(key, codePrefix) && len(key) == common.HashLength+len(codePrefix) { + return true, key[len(codePrefix):] + } + return false, nil +} + // configKey = configPrefix + hash func configKey(hash common.Hash) []byte { return append(configPrefix, hash.Bytes()...) diff --git a/core/rawdb/table.go b/core/rawdb/table.go index f9078e8..323ef62 100644 --- a/core/rawdb/table.go +++ b/core/rawdb/table.go @@ -17,7 +17,7 @@ package rawdb import ( - "github.com/ava-labs/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/ethdb" ) // table is a wrapper around a database that prefixes each key access with a pre- @@ -103,23 +103,16 @@ func (t *table) Delete(key []byte) error { return t.db.Delete(append([]byte(t.prefix), key...)) } -// NewIterator creates a binary-alphabetical iterator over the entire keyspace -// contained within the database. -func (t *table) NewIterator() ethdb.Iterator { - return t.NewIteratorWithPrefix(nil) -} - -// NewIteratorWithStart creates a binary-alphabetical iterator over a subset of -// database content starting at a particular initial key (or after, if it does -// not exist). -func (t *table) NewIteratorWithStart(start []byte) ethdb.Iterator { - return t.db.NewIteratorWithStart(start) -} - -// NewIteratorWithPrefix creates a binary-alphabetical iterator over a subset -// of database content with a particular key prefix. -func (t *table) NewIteratorWithPrefix(prefix []byte) ethdb.Iterator { - return t.db.NewIteratorWithPrefix(append([]byte(t.prefix), prefix...)) +// NewIterator creates a binary-alphabetical iterator over a subset +// of database content with a particular key prefix, starting at a particular +// initial key (or after, if it does not exist). +func (t *table) NewIterator(prefix []byte, start []byte) ethdb.Iterator { + innerPrefix := append([]byte(t.prefix), prefix...) + iter := t.db.NewIterator(innerPrefix, start) + return &tableIterator{ + iter: iter, + prefix: t.prefix, + } } // Stat returns a particular internal stat of the database. @@ -198,7 +191,69 @@ func (b *tableBatch) Reset() { b.batch.Reset() } +// tableReplayer is a wrapper around a batch replayer which truncates +// the added prefix. +type tableReplayer struct { + w ethdb.KeyValueWriter + prefix string +} + +// Put implements the interface KeyValueWriter. +func (r *tableReplayer) Put(key []byte, value []byte) error { + trimmed := key[len(r.prefix):] + return r.w.Put(trimmed, value) +} + +// Delete implements the interface KeyValueWriter. +func (r *tableReplayer) Delete(key []byte) error { + trimmed := key[len(r.prefix):] + return r.w.Delete(trimmed) +} + // Replay replays the batch contents. func (b *tableBatch) Replay(w ethdb.KeyValueWriter) error { - return b.batch.Replay(w) + return b.batch.Replay(&tableReplayer{w: w, prefix: b.prefix}) +} + +// tableIterator is a wrapper around a database iterator that prefixes each key access +// with a pre-configured string. +type tableIterator struct { + iter ethdb.Iterator + prefix string +} + +// Next moves the iterator to the next key/value pair. It returns whether the +// iterator is exhausted. +func (iter *tableIterator) Next() bool { + return iter.iter.Next() +} + +// Error returns any accumulated error. Exhausting all the key/value pairs +// is not considered to be an error. +func (iter *tableIterator) Error() error { + return iter.iter.Error() +} + +// Key returns the key of the current key/value pair, or nil if done. The caller +// should not modify the contents of the returned slice, and its contents may +// change on the next call to Next. +func (iter *tableIterator) Key() []byte { + key := iter.iter.Key() + if key == nil { + return nil + } + return key[len(iter.prefix):] +} + +// Value returns the value of the current key/value pair, or nil if done. The +// caller should not modify the contents of the returned slice, and its contents +// may change on the next call to Next. +func (iter *tableIterator) Value() []byte { + return iter.iter.Value() +} + +// Release releases associated resources. Release should always succeed and can +// be called multiple times without causing error. +func (iter *tableIterator) Release() { + iter.iter.Release() } -- cgit v1.2.3-70-g09d2 From fec21d6c958264ff5ff729c0c91e4a882aecc23b Mon Sep 17 00:00:00 2001 From: Determinant Date: Wed, 16 Sep 2020 00:39:05 -0400 Subject: add lists to keep track of actually hacked files --- copied-list.txt | 107 +++++++ core/events.go | 7 +- core/evm.go | 38 ++- core/genesis.go | 78 +++-- core/rawdb/accessors_chain.go | 238 ++++++++++++--- core/state/journal.go | 10 +- core/state/state_object.go | 161 +++++++--- core/state/statedb.go | 682 +++++++++++++++++++++++++----------------- core/state_processor.go | 22 +- hacked-list.txt | 44 +++ 10 files changed, 977 insertions(+), 410 deletions(-) create mode 100644 copied-list.txt create mode 100644 hacked-list.txt (limited to 'core/rawdb') diff --git a/copied-list.txt b/copied-list.txt new file mode 100644 index 0000000..94353ff --- /dev/null +++ b/copied-list.txt @@ -0,0 +1,107 @@ +./consensus/clique/api.go +./consensus/clique/snapshot.go +./consensus/errors.go +./consensus/ethash/algorithm.go +./consensus/ethash/api.go +./consensus/ethash/ethash.go +./consensus/ethash/sealer.go +./consensus/misc/dao.go +./consensus/misc/forks.go +./core/blockchain_insert.go +./core/blocks.go +./core/block_validator.go +./core/bloombits/doc.go +./core/bloombits/generator.go +./core/bloombits/matcher.go +./core/bloombits/scheduler.go +./core/chain_indexer.go +./core/error.go +./core/gaspool.go +./core/genesis_alloc.go +./core/headerchain.go +./core/mkalloc.go +./core/rawdb/accessors_indexes.go +./core/rawdb/accessors_metadata.go +./core/rawdb/database.go +./core/rawdb/freezer.go +./core/rawdb/freezer_reinit.go +./core/rawdb/freezer_table.go +./core/rawdb/schema.go +./core/rawdb/table.go +./core/state/database.go +./core/state/dump.go +./core/state/iterator.go +./core/state_prefetcher.go +./core/state/sync.go +./core/state_transition.go +./core/tx_cacher.go +./core/tx_journal.go +./core/tx_list.go +./core/tx_noncer.go +./core/types/bloom9.go +./core/types/derive_sha.go +./core/types/gen_header_json.go +./core/types/gen_log_json.go +./core/types/gen_receipt_json.go +./core/types/gen_tx_json.go +./core/types.go +./core/types/log.go +./core/types/receipt.go +./core/types/transaction.go +./core/types/transaction_signing.go +./core/vm/analysis.go +./core/vm/common.go +./core/vm/contract.go +./core/vm/contracts.go +./core/vm/doc.go +./core/vm/eips.go +./core/vm/gas.go +./core/vm/gas_table.go +./core/vm/gen_structlog.go +./core/vm/intpool.go +./core/vm/int_pool_verifier_empty.go +./core/vm/int_pool_verifier.go +./core/vm/logger.go +./core/vm/logger_json.go +./core/vm/memory.go +./core/vm/stack.go +./core/vm/stack_table.go +./eth/bloombits.go +./eth/filters/api.go +./eth/filters/filter.go +./eth/filters/filter_system.go +./eth/gasprice/gasprice.go +./eth/metrics.go +./eth/protocol.go +./ethstats/ethstats.go +./eth/tracers/internal/tracers/tracers.go +./eth/tracers/tracer.go +./eth/tracers/tracers.go +./internal/debug/api.go +./internal/debug/flags.go +./internal/debug/loudpanic_fallback.go +./internal/debug/loudpanic.go +./internal/debug/trace_fallback.go +./internal/debug/trace.go +./internal/ethapi/addrlock.go +./miner/unconfirmed.go +./node/defaults.go +./node/errors.go +./params/config.go +./params/dao.go +./params/denomination.go +./params/network_params.go +./params/version.go +./rpc/constants_unix_nocgo.go +./rpc/doc.go +./rpc/errors.go +./rpc/gzip.go +./rpc/handler.go +./rpc/http.go +./rpc/ipc_js.go +./rpc/json.go +./rpc/server.go +./rpc/service.go +./rpc/subscription.go +./rpc/websocket.go +./accounts/* diff --git a/core/events.go b/core/events.go index f05e69b..28fbc44 100644 --- a/core/events.go +++ b/core/events.go @@ -18,7 +18,7 @@ package core import ( "github.com/ava-labs/coreth/core/types" - "github.com/ava-labs/go-ethereum/common" + "github.com/ethereum/go-ethereum/common" ) // NewTxsEvent is posted when a batch of transactions enter the transaction pool. @@ -27,11 +27,6 @@ type NewTxsEvent struct{ Txs []*types.Transaction } // NewTxPoolHeadEvent is posted when the pool head is updated. type NewTxPoolHeadEvent struct{ Block *types.Block } -// PendingLogsEvent is posted pre mining and notifies of pending logs. -type PendingLogsEvent struct { - Logs []*types.Log -} - // NewMinedBlockEvent is posted when a block has been imported. type NewMinedBlockEvent struct{ Block *types.Block } diff --git a/core/evm.go b/core/evm.go index 796b312..74891d7 100644 --- a/core/evm.go +++ b/core/evm.go @@ -22,8 +22,8 @@ import ( "github.com/ava-labs/coreth/consensus" "github.com/ava-labs/coreth/core/types" "github.com/ava-labs/coreth/core/vm" - "github.com/ava-labs/go-ethereum/common" - "github.com/ava-labs/go-ethereum/log" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" ) // ChainContext supports retrieving headers and consensus parameters from the @@ -63,24 +63,32 @@ func NewEVMContext(msg Message, header *types.Header, chain ChainContext, author // GetHashFn returns a GetHashFunc which retrieves header hashes by number func GetHashFn(ref *types.Header, chain ChainContext) func(n uint64) common.Hash { - var cache map[uint64]common.Hash + // Cache will initially contain [refHash.parent], + // Then fill up with [refHash.p, refHash.pp, refHash.ppp, ...] + var cache []common.Hash return func(n uint64) common.Hash { // If there's no hash cache yet, make one - if cache == nil { - cache = map[uint64]common.Hash{ - ref.Number.Uint64() - 1: ref.ParentHash, - } + if len(cache) == 0 { + cache = append(cache, ref.ParentHash) } - // Try to fulfill the request from the cache - if hash, ok := cache[n]; ok { - return hash + if idx := ref.Number.Uint64() - n - 1; idx < uint64(len(cache)) { + return cache[idx] } - // Not cached, iterate the blocks and cache the hashes - for header := chain.GetHeader(ref.ParentHash, ref.Number.Uint64()-1); header != nil; header = chain.GetHeader(header.ParentHash, header.Number.Uint64()-1) { - cache[header.Number.Uint64()-1] = header.ParentHash - if n == header.Number.Uint64()-1 { - return header.ParentHash + // No luck in the cache, but we can start iterating from the last element we already know + lastKnownHash := cache[len(cache)-1] + lastKnownNumber := ref.Number.Uint64() - uint64(len(cache)) + + for { + header := chain.GetHeader(lastKnownHash, lastKnownNumber) + if header == nil { + break + } + cache = append(cache, header.ParentHash) + lastKnownHash = header.ParentHash + lastKnownNumber = header.Number.Uint64() - 1 + if n == lastKnownNumber { + return lastKnownHash } } return common.Hash{} diff --git a/core/genesis.go b/core/genesis.go index 7d21d00..e48f411 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -29,13 +29,14 @@ import ( "github.com/ava-labs/coreth/core/state" "github.com/ava-labs/coreth/core/types" "github.com/ava-labs/coreth/params" - "github.com/ava-labs/go-ethereum/common" - "github.com/ava-labs/go-ethereum/common/hexutil" - "github.com/ava-labs/go-ethereum/common/math" - "github.com/ava-labs/go-ethereum/crypto" - "github.com/ava-labs/go-ethereum/ethdb" - "github.com/ava-labs/go-ethereum/log" - "github.com/ava-labs/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/common/math" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" ) //go:generate gencodec -type Genesis -field-override genesisSpecMarshaling -out gen_genesis.go @@ -155,10 +156,6 @@ func (e *GenesisMismatchError) Error() string { // // The returned chain configuration is never nil. func SetupGenesisBlock(db ethdb.Database, genesis *Genesis) (*params.ChainConfig, common.Hash, error) { - return SetupGenesisBlockWithOverride(db, genesis, nil) -} - -func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, overrideIstanbul *big.Int) (*params.ChainConfig, common.Hash, error) { if genesis != nil && genesis.Config == nil { return params.AllEthashProtocolChanges, common.Hash{}, errGenesisNoConfig } @@ -181,7 +178,7 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override // We have the genesis block in database(perhaps in ancient database) // but the corresponding state is missing. header := rawdb.ReadHeader(db, stored, 0) - if _, err := state.New(header.Root, state.NewDatabaseWithCache(db, 0)); err != nil { + if _, err := state.New(header.Root, state.NewDatabaseWithCache(db, 0, ""), nil); err != nil { if genesis == nil { genesis = DefaultGenesisBlock() } @@ -207,8 +204,8 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override // Get the existing chain configuration. newcfg := configOrDefault(genesis, stored) - if overrideIstanbul != nil { - newcfg.IstanbulBlock = overrideIstanbul + if err := newcfg.CheckConfigForkOrder(); err != nil { + return newcfg, common.Hash{}, err } storedcfg := rawdb.ReadChainConfig(db, stored) if storedcfg == nil { @@ -243,8 +240,14 @@ func configOrDefault(g *Genesis, ghash common.Hash) *params.ChainConfig { return g.Config case ghash == params.MainnetGenesisHash: return params.MainnetChainConfig - case ghash == params.TestnetGenesisHash: - return params.TestnetChainConfig + case ghash == params.RopstenGenesisHash: + return params.RopstenChainConfig + case ghash == params.RinkebyGenesisHash: + return params.RinkebyChainConfig + case ghash == params.GoerliGenesisHash: + return params.GoerliChainConfig + case ghash == params.YoloV1GenesisHash: + return params.YoloV1ChainConfig default: return params.AllEthashProtocolChanges } @@ -256,7 +259,7 @@ func (g *Genesis) ToBlock(db ethdb.Database) *types.Block { if db == nil { db = rawdb.NewMemoryDatabase() } - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(db), nil) for addr, account := range g.Alloc { statedb.AddBalance(addr, account.Balance) statedb.SetCode(addr, account.Code) @@ -292,9 +295,9 @@ func (g *Genesis) ToBlock(db ethdb.Database) *types.Block { head.Difficulty = params.GenesisDifficulty } statedb.Commit(false) - statedb.Database().TrieDB().Commit(root, true) + statedb.Database().TrieDB().Commit(root, true, nil) - return types.NewBlock(head, nil, nil, nil, nil) + return types.NewBlock(head, nil, nil, nil, new(trie.Trie), nil) } // Commit writes the block and state of a genesis specification to the database. @@ -304,6 +307,13 @@ func (g *Genesis) Commit(db ethdb.Database) (*types.Block, error) { if block.Number().Sign() != 0 { return nil, fmt.Errorf("can't commit genesis block with number > 0") } + config := g.Config + if config == nil { + config = params.AllEthashProtocolChanges + } + if err := config.CheckConfigForkOrder(); err != nil { + return nil, err + } rawdb.WriteTd(db, block.Hash(), block.NumberU64(), g.Difficulty) rawdb.WriteBlock(db, block) rawdb.WriteReceipts(db, block.Hash(), block.NumberU64(), nil) @@ -311,11 +321,6 @@ func (g *Genesis) Commit(db ethdb.Database) (*types.Block, error) { rawdb.WriteHeadBlockHash(db, block.Hash()) rawdb.WriteHeadFastBlockHash(db, block.Hash()) rawdb.WriteHeadHeaderHash(db, block.Hash()) - - config := g.Config - if config == nil { - config = params.AllEthashProtocolChanges - } rawdb.WriteChainConfig(db, block.Hash(), config) return block, nil } @@ -348,15 +353,15 @@ func DefaultGenesisBlock() *Genesis { } } -// DefaultTestnetGenesisBlock returns the Ropsten network genesis block. -func DefaultTestnetGenesisBlock() *Genesis { +// DefaultRopstenGenesisBlock returns the Ropsten network genesis block. +func DefaultRopstenGenesisBlock() *Genesis { return &Genesis{ - Config: params.TestnetChainConfig, + Config: params.RopstenChainConfig, Nonce: 66, ExtraData: hexutil.MustDecode("0x3535353535353535353535353535353535353535353535353535353535353535"), GasLimit: 16777216, Difficulty: big.NewInt(1048576), - Alloc: decodePrealloc(testnetAllocData), + Alloc: decodePrealloc(ropstenAllocData), } } @@ -384,8 +389,18 @@ func DefaultGoerliGenesisBlock() *Genesis { } } -// DeveloperGenesisBlock returns the 'geth --dev' genesis block. Note, this must -// be seeded with the +func DefaultYoloV1GenesisBlock() *Genesis { + return &Genesis{ + Config: params.YoloV1ChainConfig, + Timestamp: 0x5ed754f1, + ExtraData: hexutil.MustDecode("0x00000000000000000000000000000000000000000000000000000000000000008a37866fd3627c9205a37c8685666f32ec07bb1b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), + GasLimit: 0x47b760, + Difficulty: big.NewInt(1), + Alloc: decodePrealloc(yoloV1AllocData), + } +} + +// DeveloperGenesisBlock returns the 'geth --dev' genesis block. func DeveloperGenesisBlock(period uint64, faucet common.Address) *Genesis { // Override the default period to the user requested one config := *params.AllCliqueProtocolChanges @@ -395,7 +410,7 @@ func DeveloperGenesisBlock(period uint64, faucet common.Address) *Genesis { return &Genesis{ Config: &config, ExtraData: append(append(make([]byte, 32), faucet[:]...), make([]byte, crypto.SignatureLength)...), - GasLimit: 6283185, + GasLimit: 11500000, Difficulty: big.NewInt(1), Alloc: map[common.Address]GenesisAccount{ common.BytesToAddress([]byte{1}): {Balance: big.NewInt(1)}, // ECRecover @@ -406,6 +421,7 @@ func DeveloperGenesisBlock(period uint64, faucet common.Address) *Genesis { common.BytesToAddress([]byte{6}): {Balance: big.NewInt(1)}, // ECAdd common.BytesToAddress([]byte{7}): {Balance: big.NewInt(1)}, // ECScalarMul common.BytesToAddress([]byte{8}): {Balance: big.NewInt(1)}, // ECPairing + common.BytesToAddress([]byte{9}): {Balance: big.NewInt(1)}, // BLAKE2b faucet: {Balance: new(big.Int).Sub(new(big.Int).Lsh(big.NewInt(1), 256), big.NewInt(9))}, }, } diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index fdfd6ec..cd48885 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -23,10 +23,11 @@ import ( "github.com/ava-labs/coreth/core/types" "github.com/ava-labs/coreth/params" - "github.com/ava-labs/go-ethereum/common" - "github.com/ava-labs/go-ethereum/ethdb" - "github.com/ava-labs/go-ethereum/log" - "github.com/ava-labs/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" ) // ReadCanonicalHash retrieves the hash assigned to a canonical block number. @@ -68,7 +69,7 @@ func ReadAllHashes(db ethdb.Iteratee, number uint64) []common.Hash { prefix := headerKeyPrefix(number) hashes := make([]common.Hash, 0, 1) - it := db.NewIteratorWithPrefix(prefix) + it := db.NewIterator(prefix, nil) defer it.Release() for it.Next() { @@ -79,6 +80,39 @@ func ReadAllHashes(db ethdb.Iteratee, number uint64) []common.Hash { return hashes } +// ReadAllCanonicalHashes retrieves all canonical number and hash mappings at the +// certain chain range. If the accumulated entries reaches the given threshold, +// abort the iteration and return the semi-finish result. +func ReadAllCanonicalHashes(db ethdb.Iteratee, from uint64, to uint64, limit int) ([]uint64, []common.Hash) { + // Short circuit if the limit is 0. + if limit == 0 { + return nil, nil + } + var ( + numbers []uint64 + hashes []common.Hash + ) + // Construct the key prefix of start point. + start, end := headerHashKey(from), headerHashKey(to) + it := db.NewIterator(nil, start) + defer it.Release() + + for it.Next() { + if bytes.Compare(it.Key(), end) >= 0 { + break + } + if key := it.Key(); len(key) == len(headerPrefix)+8+1 && bytes.Equal(key[len(key)-1:], headerHashSuffix) { + numbers = append(numbers, binary.BigEndian.Uint64(key[len(headerPrefix):len(headerPrefix)+8])) + hashes = append(hashes, common.BytesToHash(it.Value())) + // If the accumulated entries reaches the limit threshold, return. + if len(numbers) >= limit { + break + } + } + } + return numbers, hashes +} + // ReadHeaderNumber returns the header number assigned to a hash. func ReadHeaderNumber(db ethdb.KeyValueReader, hash common.Hash) *uint64 { data, _ := db.Get(headerNumberKey(hash)) @@ -153,6 +187,32 @@ func WriteHeadFastBlockHash(db ethdb.KeyValueWriter, hash common.Hash) { } } +// ReadLastPivotNumber retrieves the number of the last pivot block. If the node +// full synced, the last pivot will always be nil. +func ReadLastPivotNumber(db ethdb.KeyValueReader) *uint64 { + data, _ := db.Get(lastPivotKey) + if len(data) == 0 { + return nil + } + var pivot uint64 + if err := rlp.DecodeBytes(data, &pivot); err != nil { + log.Error("Invalid pivot block number in database", "err", err) + return nil + } + return &pivot +} + +// WriteLastPivotNumber stores the number of the last pivot block. +func WriteLastPivotNumber(db ethdb.KeyValueWriter, pivot uint64) { + enc, err := rlp.EncodeToBytes(pivot) + if err != nil { + log.Crit("Failed to encode pivot block number", "err", err) + } + if err := db.Put(lastPivotKey, enc); err != nil { + log.Crit("Failed to store pivot block number", "err", err) + } +} + // ReadFastTrieProgress retrieves the number of tries nodes fast synced to allow // reporting correct numbers across restarts. func ReadFastTrieProgress(db ethdb.KeyValueReader) uint64 { @@ -171,20 +231,66 @@ func WriteFastTrieProgress(db ethdb.KeyValueWriter, count uint64) { } } +// ReadTxIndexTail retrieves the number of oldest indexed block +// whose transaction indices has been indexed. If the corresponding entry +// is non-existent in database it means the indexing has been finished. +func ReadTxIndexTail(db ethdb.KeyValueReader) *uint64 { + data, _ := db.Get(txIndexTailKey) + if len(data) != 8 { + return nil + } + number := binary.BigEndian.Uint64(data) + return &number +} + +// WriteTxIndexTail stores the number of oldest indexed block +// into database. +func WriteTxIndexTail(db ethdb.KeyValueWriter, number uint64) { + if err := db.Put(txIndexTailKey, encodeBlockNumber(number)); err != nil { + log.Crit("Failed to store the transaction index tail", "err", err) + } +} + +// ReadFastTxLookupLimit retrieves the tx lookup limit used in fast sync. +func ReadFastTxLookupLimit(db ethdb.KeyValueReader) *uint64 { + data, _ := db.Get(fastTxLookupLimitKey) + if len(data) != 8 { + return nil + } + number := binary.BigEndian.Uint64(data) + return &number +} + +// WriteFastTxLookupLimit stores the txlookup limit used in fast sync into database. +func WriteFastTxLookupLimit(db ethdb.KeyValueWriter, number uint64) { + if err := db.Put(fastTxLookupLimitKey, encodeBlockNumber(number)); err != nil { + log.Crit("Failed to store transaction lookup limit for fast sync", "err", err) + } +} + // ReadHeaderRLP retrieves a block header in its raw RLP database encoding. func ReadHeaderRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue { + // First try to look up the data in ancient database. Extra hash + // comparison is necessary since ancient database only maintains + // the canonical data. data, _ := db.Ancient(freezerHeaderTable, number) - if len(data) == 0 { - data, _ = db.Get(headerKey(number, hash)) - // In the background freezer is moving data from leveldb to flatten files. - // So during the first check for ancient db, the data is not yet in there, - // but when we reach into leveldb, the data was already moved. That would - // result in a not found error. - if len(data) == 0 { - data, _ = db.Ancient(freezerHeaderTable, number) - } + if len(data) > 0 && crypto.Keccak256Hash(data) == hash { + return data } - return data + // Then try to look up the data in leveldb. + data, _ = db.Get(headerKey(number, hash)) + if len(data) > 0 { + return data + } + // In the background freezer is moving data from leveldb to flatten files. + // So during the first check for ancient db, the data is not yet in there, + // but when we reach into leveldb, the data was already moved. That would + // result in a not found error. + data, _ = db.Ancient(freezerHeaderTable, number) + if len(data) > 0 && crypto.Keccak256Hash(data) == hash { + return data + } + return nil // Can't find the data anywhere. } // HasHeader verifies the existence of a block header corresponding to the hash. @@ -251,9 +357,43 @@ func deleteHeaderWithoutNumber(db ethdb.KeyValueWriter, hash common.Hash, number // ReadBodyRLP retrieves the block body (transactions and uncles) in RLP encoding. func ReadBodyRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue { + // First try to look up the data in ancient database. Extra hash + // comparison is necessary since ancient database only maintains + // the canonical data. + data, _ := db.Ancient(freezerBodiesTable, number) + if len(data) > 0 { + h, _ := db.Ancient(freezerHashTable, number) + if common.BytesToHash(h) == hash { + return data + } + } + // Then try to look up the data in leveldb. + data, _ = db.Get(blockBodyKey(number, hash)) + if len(data) > 0 { + return data + } + // In the background freezer is moving data from leveldb to flatten files. + // So during the first check for ancient db, the data is not yet in there, + // but when we reach into leveldb, the data was already moved. That would + // result in a not found error. + data, _ = db.Ancient(freezerBodiesTable, number) + if len(data) > 0 { + h, _ := db.Ancient(freezerHashTable, number) + if common.BytesToHash(h) == hash { + return data + } + } + return nil // Can't find the data anywhere. +} + +// ReadCanonicalBodyRLP retrieves the block body (transactions and uncles) for the canonical +// block at number, in RLP encoding. +func ReadCanonicalBodyRLP(db ethdb.Reader, number uint64) rlp.RawValue { + // If it's an ancient one, we don't need the canonical hash data, _ := db.Ancient(freezerBodiesTable, number) if len(data) == 0 { - data, _ = db.Get(blockBodyKey(number, hash)) + // Need to get the hash + data, _ = db.Get(blockBodyKey(number, ReadCanonicalHash(db, number))) // In the background freezer is moving data from leveldb to flatten files. // So during the first check for ancient db, the data is not yet in there, // but when we reach into leveldb, the data was already moved. That would @@ -315,18 +455,33 @@ func DeleteBody(db ethdb.KeyValueWriter, hash common.Hash, number uint64) { // ReadTdRLP retrieves a block's total difficulty corresponding to the hash in RLP encoding. func ReadTdRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue { + // First try to look up the data in ancient database. Extra hash + // comparison is necessary since ancient database only maintains + // the canonical data. data, _ := db.Ancient(freezerDifficultyTable, number) - if len(data) == 0 { - data, _ = db.Get(headerTDKey(number, hash)) - // In the background freezer is moving data from leveldb to flatten files. - // So during the first check for ancient db, the data is not yet in there, - // but when we reach into leveldb, the data was already moved. That would - // result in a not found error. - if len(data) == 0 { - data, _ = db.Ancient(freezerDifficultyTable, number) + if len(data) > 0 { + h, _ := db.Ancient(freezerHashTable, number) + if common.BytesToHash(h) == hash { + return data } } - return data + // Then try to look up the data in leveldb. + data, _ = db.Get(headerTDKey(number, hash)) + if len(data) > 0 { + return data + } + // In the background freezer is moving data from leveldb to flatten files. + // So during the first check for ancient db, the data is not yet in there, + // but when we reach into leveldb, the data was already moved. That would + // result in a not found error. + data, _ = db.Ancient(freezerDifficultyTable, number) + if len(data) > 0 { + h, _ := db.Ancient(freezerHashTable, number) + if common.BytesToHash(h) == hash { + return data + } + } + return nil // Can't find the data anywhere. } // ReadTd retrieves a block's total difficulty corresponding to the hash. @@ -375,18 +530,33 @@ func HasReceipts(db ethdb.Reader, hash common.Hash, number uint64) bool { // ReadReceiptsRLP retrieves all the transaction receipts belonging to a block in RLP encoding. func ReadReceiptsRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue { + // First try to look up the data in ancient database. Extra hash + // comparison is necessary since ancient database only maintains + // the canonical data. data, _ := db.Ancient(freezerReceiptTable, number) - if len(data) == 0 { - data, _ = db.Get(blockReceiptsKey(number, hash)) - // In the background freezer is moving data from leveldb to flatten files. - // So during the first check for ancient db, the data is not yet in there, - // but when we reach into leveldb, the data was already moved. That would - // result in a not found error. - if len(data) == 0 { - data, _ = db.Ancient(freezerReceiptTable, number) + if len(data) > 0 { + h, _ := db.Ancient(freezerHashTable, number) + if common.BytesToHash(h) == hash { + return data } } - return data + // Then try to look up the data in leveldb. + data, _ = db.Get(blockReceiptsKey(number, hash)) + if len(data) > 0 { + return data + } + // In the background freezer is moving data from leveldb to flatten files. + // So during the first check for ancient db, the data is not yet in there, + // but when we reach into leveldb, the data was already moved. That would + // result in a not found error. + data, _ = db.Ancient(freezerReceiptTable, number) + if len(data) > 0 { + h, _ := db.Ancient(freezerHashTable, number) + if common.BytesToHash(h) == hash { + return data + } + } + return nil // Can't find the data anywhere. } // ReadRawReceipts retrieves all the transaction receipts belonging to a block. diff --git a/core/state/journal.go b/core/state/journal.go index 6e85173..cfa1a4a 100644 --- a/core/state/journal.go +++ b/core/state/journal.go @@ -90,7 +90,8 @@ type ( account *common.Address } resetObjectChange struct { - prev *stateObject + prev *stateObject + prevdestruct bool } suicideChange struct { account *common.Address @@ -130,9 +131,7 @@ type ( hash common.Hash } touchChange struct { - account *common.Address - prev bool - prevDirty bool + account *common.Address } ) @@ -147,6 +146,9 @@ func (ch createObjectChange) dirtied() *common.Address { func (ch resetObjectChange) revert(s *StateDB) { s.setStateObject(ch.prev) + if !ch.prevdestruct && s.snap != nil { + delete(s.snapDestructs, ch.prev.addrHash) + } } func (ch resetObjectChange) dirtied() *common.Address { diff --git a/core/state/state_object.go b/core/state/state_object.go index 9c47dc4..2893f80 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -23,10 +23,10 @@ import ( "math/big" "time" - "github.com/ava-labs/go-ethereum/common" - "github.com/ava-labs/go-ethereum/crypto" - "github.com/ava-labs/go-ethereum/metrics" - "github.com/ava-labs/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/rlp" ) var emptyCodeHash = crypto.Keccak256(nil) @@ -79,9 +79,10 @@ type stateObject struct { trie Trie // storage trie, which becomes non-nil on first access code Code // contract bytecode, which gets set when code is loaded - originStorage Storage // Storage cache of original entries to dedup rewrites - dirtyStorage Storage // Storage entries that need to be flushed to disk - fakeStorage Storage // Fake storage which constructed by caller for debugging purpose. + originStorage Storage // Storage cache of original entries to dedup rewrites, reset for every transaction + pendingStorage Storage // Storage entries that need to be flushed to disk, at the end of an entire block + dirtyStorage Storage // Storage entries that have been modified in the current transaction execution + fakeStorage Storage // Fake storage which constructed by caller for debugging purpose. // Cache flags. // When an object is marked suicided it will be delete from the trie @@ -114,13 +115,17 @@ func newObject(db *StateDB, address common.Address, data Account) *stateObject { if data.CodeHash == nil { data.CodeHash = emptyCodeHash } + if data.Root == (common.Hash{}) { + data.Root = emptyRoot + } return &stateObject{ - db: db, - address: address, - addrHash: crypto.Keccak256Hash(address[:]), - data: data, - originStorage: make(Storage), - dirtyStorage: make(Storage), + db: db, + address: address, + addrHash: crypto.Keccak256Hash(address[:]), + data: data, + originStorage: make(Storage), + pendingStorage: make(Storage), + dirtyStorage: make(Storage), } } @@ -184,21 +189,44 @@ func (s *stateObject) GetCommittedState(db Database, key common.Hash) common.Has if s.fakeStorage != nil { return s.fakeStorage[key] } - // If we have the original value cached, return that - value, cached := s.originStorage[key] - if cached { + // If we have a pending write or clean cached, return that + if value, pending := s.pendingStorage[key]; pending { return value } - // Track the amount of time wasted on reading the storage trie - if metrics.EnabledExpensive { - defer func(start time.Time) { s.db.StorageReads += time.Since(start) }(time.Now()) + if value, cached := s.originStorage[key]; cached { + return value } - // Otherwise load the value from the database - enc, err := s.getTrie(db).TryGet(key[:]) - if err != nil { - s.setError(err) - return common.Hash{} + // If no live objects are available, attempt to use snapshots + var ( + enc []byte + err error + ) + if s.db.snap != nil { + if metrics.EnabledExpensive { + defer func(start time.Time) { s.db.SnapshotStorageReads += time.Since(start) }(time.Now()) + } + // If the object was destructed in *this* block (and potentially resurrected), + // the storage has been cleared out, and we should *not* consult the previous + // snapshot about any storage values. The only possible alternatives are: + // 1) resurrect happened, and new slot values were set -- those should + // have been handles via pendingStorage above. + // 2) we don't have new values, and can deliver empty response back + if _, destructed := s.db.snapDestructs[s.addrHash]; destructed { + return common.Hash{} + } + enc, err = s.db.snap.Storage(s.addrHash, crypto.Keccak256Hash(key.Bytes())) + } + // If snapshot unavailable or reading from it failed, load from the database + if s.db.snap == nil || err != nil { + if metrics.EnabledExpensive { + defer func(start time.Time) { s.db.StorageReads += time.Since(start) }(time.Now()) + } + if enc, err = s.getTrie(db).TryGet(key.Bytes()); err != nil { + s.setError(err) + return common.Hash{} + } } + var value common.Hash if len(enc) > 0 { _, content, _, err := rlp.Split(enc) if err != nil { @@ -253,38 +281,73 @@ func (s *stateObject) setState(key, value common.Hash) { s.dirtyStorage[key] = value } +// finalise moves all dirty storage slots into the pending area to be hashed or +// committed later. It is invoked at the end of every transaction. +func (s *stateObject) finalise() { + for key, value := range s.dirtyStorage { + s.pendingStorage[key] = value + } + if len(s.dirtyStorage) > 0 { + s.dirtyStorage = make(Storage) + } +} + // updateTrie writes cached storage modifications into the object's storage trie. +// It will return nil if the trie has not been loaded and no changes have been made func (s *stateObject) updateTrie(db Database) Trie { + // Make sure all dirty slots are finalized into the pending storage area + s.finalise() + if len(s.pendingStorage) == 0 { + return s.trie + } // Track the amount of time wasted on updating the storge trie if metrics.EnabledExpensive { defer func(start time.Time) { s.db.StorageUpdates += time.Since(start) }(time.Now()) } - // Update all the dirty slots in the trie + // Retrieve the snapshot storage map for the object + var storage map[common.Hash][]byte + if s.db.snap != nil { + // Retrieve the old storage map, if available, create a new one otherwise + storage = s.db.snapStorage[s.addrHash] + if storage == nil { + storage = make(map[common.Hash][]byte) + s.db.snapStorage[s.addrHash] = storage + } + } + // Insert all the pending updates into the trie tr := s.getTrie(db) - for key, value := range s.dirtyStorage { - delete(s.dirtyStorage, key) - + for key, value := range s.pendingStorage { // Skip noop changes, persist actual changes if value == s.originStorage[key] { continue } s.originStorage[key] = value + var v []byte if (value == common.Hash{}) { s.setError(tr.TryDelete(key[:])) - continue + } else { + // Encoding []byte cannot fail, ok to ignore the error. + v, _ = rlp.EncodeToBytes(common.TrimLeftZeroes(value[:])) + s.setError(tr.TryUpdate(key[:], v)) + } + // If state snapshotting is active, cache the data til commit + if storage != nil { + storage[crypto.Keccak256Hash(key[:])] = v // v will be nil if value is 0x00 } - // Encoding []byte cannot fail, ok to ignore the error. - v, _ := rlp.EncodeToBytes(bytes.TrimLeft(value[:], "\x00")) - s.setError(tr.TryUpdate(key[:], v)) + } + if len(s.pendingStorage) > 0 { + s.pendingStorage = make(Storage) } return tr } // UpdateRoot sets the trie root to the current root hash of func (s *stateObject) updateRoot(db Database) { - s.updateTrie(db) - + // If nothing changed, don't bother with hashing anything + if s.updateTrie(db) == nil { + return + } // Track the amount of time wasted on hashing the storge trie if metrics.EnabledExpensive { defer func(start time.Time) { s.db.StorageHashes += time.Since(start) }(time.Now()) @@ -295,7 +358,10 @@ func (s *stateObject) updateRoot(db Database) { // CommitTrie the storage trie of the object to db. // This updates the trie root. func (s *stateObject) CommitTrie(db Database) error { - s.updateTrie(db) + // If nothing changed, don't bother with hashing anything + if s.updateTrie(db) == nil { + return nil + } if s.dbErr != nil { return s.dbErr } @@ -310,22 +376,21 @@ func (s *stateObject) CommitTrie(db Database) error { return err } -// AddBalance removes amount from c's balance. +// AddBalance adds amount to s's balance. // It is used to add funds to the destination account of a transfer. func (s *stateObject) AddBalance(amount *big.Int) { - // EIP158: We must check emptiness for the objects such that the account + // EIP161: We must check emptiness for the objects such that the account // clearing (0,0,0 objects) can take effect. if amount.Sign() == 0 { if s.empty() { s.touch() } - return } s.SetBalance(new(big.Int).Add(s.Balance(), amount)) } -// SubBalance removes amount from c's balance. +// SubBalance removes amount from s's balance. // It is used to remove funds from the origin account of a transfer. func (s *stateObject) SubBalance(amount *big.Int) { if amount.Sign() == 0 { @@ -388,6 +453,7 @@ func (s *stateObject) deepCopy(db *StateDB) *stateObject { stateObject.code = s.code stateObject.dirtyStorage = s.dirtyStorage.Copy() stateObject.originStorage = s.originStorage.Copy() + stateObject.pendingStorage = s.pendingStorage.Copy() stateObject.suicided = s.suicided stateObject.dirtyCode = s.dirtyCode stateObject.deleted = s.deleted @@ -419,6 +485,23 @@ func (s *stateObject) Code(db Database) []byte { return code } +// CodeSize returns the size of the contract code associated with this object, +// or zero if none. This method is an almost mirror of Code, but uses a cache +// inside the database to avoid loading codes seen recently. +func (s *stateObject) CodeSize(db Database) int { + if s.code != nil { + return len(s.code) + } + if bytes.Equal(s.CodeHash(), emptyCodeHash) { + return 0 + } + size, err := db.ContractCodeSize(s.addrHash, common.BytesToHash(s.CodeHash())) + if err != nil { + s.setError(fmt.Errorf("can't load code size %x: %v", s.CodeHash(), err)) + } + return size +} + func (s *stateObject) SetCode(codeHash common.Hash, code []byte) { prevcode := s.Code(s.db.db) s.db.journal.append(codeChange{ diff --git a/core/state/statedb.go b/core/state/statedb.go index 9c7535b..805c607 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -24,13 +24,15 @@ import ( "sort" "time" + "github.com/ava-labs/coreth/core/rawdb" + "github.com/ava-labs/coreth/core/state/snapshot" "github.com/ava-labs/coreth/core/types" - "github.com/ava-labs/go-ethereum/common" - "github.com/ava-labs/go-ethereum/crypto" - "github.com/ava-labs/go-ethereum/log" - "github.com/ava-labs/go-ethereum/metrics" - "github.com/ava-labs/go-ethereum/rlp" - "github.com/ava-labs/go-ethereum/trie" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" ) type revision struct { @@ -42,9 +44,6 @@ var ( // emptyRoot is the known root hash of an empty trie. emptyRoot = common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") zeroRoot = common.HexToHash("0000000000000000000000000000000000000000000000000000000000000000") - - // emptyCode is the known hash of the empty EVM bytecode. - emptyCode = crypto.Keccak256Hash(nil) ) type proofList [][]byte @@ -58,7 +57,7 @@ func (n *proofList) Delete(key []byte) error { panic("not supported") } -// StateDBs within the ethereum protocol are used to store anything +// StateDB structs within the ethereum protocol are used to store anything // within the merkle trie. StateDBs take care of caching and storing // nested states. It's the general query interface to retrieve: // * Contracts @@ -67,9 +66,16 @@ type StateDB struct { db Database trie Trie + snaps *snapshot.Tree + snap snapshot.Snapshot + snapDestructs map[common.Hash]struct{} + snapAccounts map[common.Hash][]byte + snapStorage map[common.Hash]map[common.Hash][]byte + // This map holds 'live' objects, which will get modified while processing a state transition. - stateObjects map[common.Address]*stateObject - stateObjectsDirty map[common.Address]struct{} + stateObjects map[common.Address]*stateObject + stateObjectsPending map[common.Address]struct{} // State objects finalized but not yet written to the trie + stateObjectsDirty map[common.Address]struct{} // State objects modified in the current execution // DB error. // State objects are used by the consensus core and VM which are @@ -95,134 +101,157 @@ type StateDB struct { nextRevisionId int // Measurements gathered during execution for debugging purposes - AccountReads time.Duration - AccountHashes time.Duration - AccountUpdates time.Duration - AccountCommits time.Duration - StorageReads time.Duration - StorageHashes time.Duration - StorageUpdates time.Duration - StorageCommits time.Duration -} - -// Create a new state from a given trie. -func New(root common.Hash, db Database) (*StateDB, error) { + AccountReads time.Duration + AccountHashes time.Duration + AccountUpdates time.Duration + AccountCommits time.Duration + StorageReads time.Duration + StorageHashes time.Duration + StorageUpdates time.Duration + StorageCommits time.Duration + SnapshotAccountReads time.Duration + SnapshotStorageReads time.Duration + SnapshotCommits time.Duration +} + +// New creates a new state from a given trie. +func New(root common.Hash, db Database, snaps *snapshot.Tree) (*StateDB, error) { tr, err := db.OpenTrie(root) if err != nil { return nil, err } - return &StateDB{ - db: db, - trie: tr, - stateObjects: make(map[common.Address]*stateObject), - stateObjectsDirty: make(map[common.Address]struct{}), - logs: make(map[common.Hash][]*types.Log), - preimages: make(map[common.Hash][]byte), - journal: newJournal(), - }, nil + sdb := &StateDB{ + db: db, + trie: tr, + snaps: snaps, + stateObjects: make(map[common.Address]*stateObject), + stateObjectsPending: make(map[common.Address]struct{}), + stateObjectsDirty: make(map[common.Address]struct{}), + logs: make(map[common.Hash][]*types.Log), + preimages: make(map[common.Hash][]byte), + journal: newJournal(), + } + if sdb.snaps != nil { + if sdb.snap = sdb.snaps.Snapshot(root); sdb.snap != nil { + sdb.snapDestructs = make(map[common.Hash]struct{}) + sdb.snapAccounts = make(map[common.Hash][]byte) + sdb.snapStorage = make(map[common.Hash]map[common.Hash][]byte) + } + } + return sdb, nil } // setError remembers the first non-nil error it is called with. -func (self *StateDB) setError(err error) { - if self.dbErr == nil { - self.dbErr = err +func (s *StateDB) setError(err error) { + if s.dbErr == nil { + s.dbErr = err } } -func (self *StateDB) Error() error { - return self.dbErr +func (s *StateDB) Error() error { + return s.dbErr } // Reset clears out all ephemeral state objects from the state db, but keeps // the underlying state trie to avoid reloading data for the next operations. -func (self *StateDB) Reset(root common.Hash) error { - tr, err := self.db.OpenTrie(root) +func (s *StateDB) Reset(root common.Hash) error { + tr, err := s.db.OpenTrie(root) if err != nil { return err } - self.trie = tr - self.stateObjects = make(map[common.Address]*stateObject) - self.stateObjectsDirty = make(map[common.Address]struct{}) - self.thash = common.Hash{} - self.bhash = common.Hash{} - self.txIndex = 0 - self.logs = make(map[common.Hash][]*types.Log) - self.logSize = 0 - self.preimages = make(map[common.Hash][]byte) - self.clearJournalAndRefund() + s.trie = tr + s.stateObjects = make(map[common.Address]*stateObject) + s.stateObjectsPending = make(map[common.Address]struct{}) + s.stateObjectsDirty = make(map[common.Address]struct{}) + s.thash = common.Hash{} + s.bhash = common.Hash{} + s.txIndex = 0 + s.logs = make(map[common.Hash][]*types.Log) + s.logSize = 0 + s.preimages = make(map[common.Hash][]byte) + s.clearJournalAndRefund() + + if s.snaps != nil { + s.snapAccounts, s.snapDestructs, s.snapStorage = nil, nil, nil + if s.snap = s.snaps.Snapshot(root); s.snap != nil { + s.snapDestructs = make(map[common.Hash]struct{}) + s.snapAccounts = make(map[common.Hash][]byte) + s.snapStorage = make(map[common.Hash]map[common.Hash][]byte) + } + } return nil } -func (self *StateDB) AddLog(log *types.Log) { - self.journal.append(addLogChange{txhash: self.thash}) +func (s *StateDB) AddLog(log *types.Log) { + s.journal.append(addLogChange{txhash: s.thash}) - log.TxHash = self.thash - log.BlockHash = self.bhash - log.TxIndex = uint(self.txIndex) - log.Index = self.logSize - self.logs[self.thash] = append(self.logs[self.thash], log) - self.logSize++ + log.TxHash = s.thash + log.BlockHash = s.bhash + log.TxIndex = uint(s.txIndex) + log.Index = s.logSize + s.logs[s.thash] = append(s.logs[s.thash], log) + s.logSize++ } -func (self *StateDB) GetLogs(hash common.Hash) []*types.Log { - return self.logs[hash] +func (s *StateDB) GetLogs(hash common.Hash) []*types.Log { + return s.logs[hash] } -func (self *StateDB) Logs() []*types.Log { +func (s *StateDB) Logs() []*types.Log { var logs []*types.Log - for _, lgs := range self.logs { + for _, lgs := range s.logs { logs = append(logs, lgs...) } return logs } // AddPreimage records a SHA3 preimage seen by the VM. -func (self *StateDB) AddPreimage(hash common.Hash, preimage []byte) { - if _, ok := self.preimages[hash]; !ok { - self.journal.append(addPreimageChange{hash: hash}) +func (s *StateDB) AddPreimage(hash common.Hash, preimage []byte) { + if _, ok := s.preimages[hash]; !ok { + s.journal.append(addPreimageChange{hash: hash}) pi := make([]byte, len(preimage)) copy(pi, preimage) - self.preimages[hash] = pi + s.preimages[hash] = pi } } // Preimages returns a list of SHA3 preimages that have been submitted. -func (self *StateDB) Preimages() map[common.Hash][]byte { - return self.preimages +func (s *StateDB) Preimages() map[common.Hash][]byte { + return s.preimages } // AddRefund adds gas to the refund counter -func (self *StateDB) AddRefund(gas uint64) { - self.journal.append(refundChange{prev: self.refund}) - self.refund += gas +func (s *StateDB) AddRefund(gas uint64) { + s.journal.append(refundChange{prev: s.refund}) + s.refund += gas } // SubRefund removes gas from the refund counter. // This method will panic if the refund counter goes below zero -func (self *StateDB) SubRefund(gas uint64) { - self.journal.append(refundChange{prev: self.refund}) - if gas > self.refund { - panic("Refund counter below zero") +func (s *StateDB) SubRefund(gas uint64) { + s.journal.append(refundChange{prev: s.refund}) + if gas > s.refund { + panic(fmt.Sprintf("Refund counter below zero (gas: %d > refund: %d)", gas, s.refund)) } - self.refund -= gas + s.refund -= gas } // Exist reports whether the given account address exists in the state. // Notably this also returns true for suicided accounts. -func (self *StateDB) Exist(addr common.Address) bool { - return self.getStateObject(addr) != nil +func (s *StateDB) Exist(addr common.Address) bool { + return s.getStateObject(addr) != nil } // Empty returns whether the state object is either non-existent // or empty according to the EIP161 specification (balance = nonce = code = 0) -func (self *StateDB) Empty(addr common.Address) bool { - so := self.getStateObject(addr) +func (s *StateDB) Empty(addr common.Address) bool { + so := s.getStateObject(addr) return so == nil || so.empty() } -// Retrieve the balance from the given address or 0 if object not found -func (self *StateDB) GetBalance(addr common.Address) *big.Int { - stateObject := self.getStateObject(addr) +// GetBalance retrieves the balance from the given address or 0 if object not found +func (s *StateDB) GetBalance(addr common.Address) *big.Int { + stateObject := s.getStateObject(addr) if stateObject != nil { return stateObject.Balance() } @@ -262,8 +291,8 @@ func (self *StateDB) IsMultiCoin(addr common.Address) bool { return false } -func (self *StateDB) GetNonce(addr common.Address) uint64 { - stateObject := self.getStateObject(addr) +func (s *StateDB) GetNonce(addr common.Address) uint64 { + stateObject := s.getStateObject(addr) if stateObject != nil { return stateObject.Nonce() } @@ -272,40 +301,33 @@ func (self *StateDB) GetNonce(addr common.Address) uint64 { } // TxIndex returns the current transaction index set by Prepare. -func (self *StateDB) TxIndex() int { - return self.txIndex +func (s *StateDB) TxIndex() int { + return s.txIndex } // BlockHash returns the current block hash set by Prepare. -func (self *StateDB) BlockHash() common.Hash { - return self.bhash +func (s *StateDB) BlockHash() common.Hash { + return s.bhash } -func (self *StateDB) GetCode(addr common.Address) []byte { - stateObject := self.getStateObject(addr) +func (s *StateDB) GetCode(addr common.Address) []byte { + stateObject := s.getStateObject(addr) if stateObject != nil { - return stateObject.Code(self.db) + return stateObject.Code(s.db) } return nil } -func (self *StateDB) GetCodeSize(addr common.Address) int { - stateObject := self.getStateObject(addr) - if stateObject == nil { - return 0 - } - if stateObject.code != nil { - return len(stateObject.code) - } - size, err := self.db.ContractCodeSize(stateObject.addrHash, common.BytesToHash(stateObject.CodeHash())) - if err != nil { - self.setError(err) +func (s *StateDB) GetCodeSize(addr common.Address) int { + stateObject := s.getStateObject(addr) + if stateObject != nil { + return stateObject.CodeSize(s.db) } - return size + return 0 } -func (self *StateDB) GetCodeHash(addr common.Address) common.Hash { - stateObject := self.getStateObject(addr) +func (s *StateDB) GetCodeHash(addr common.Address) common.Hash { + stateObject := s.getStateObject(addr) if stateObject == nil { return common.Hash{} } @@ -313,25 +335,25 @@ func (self *StateDB) GetCodeHash(addr common.Address) common.Hash { } // GetState retrieves a value from the given account's storage trie. -func (self *StateDB) GetState(addr common.Address, hash common.Hash) common.Hash { - stateObject := self.getStateObject(addr) +func (s *StateDB) GetState(addr common.Address, hash common.Hash) common.Hash { + stateObject := s.getStateObject(addr) if stateObject != nil { - return stateObject.GetState(self.db, hash) + return stateObject.GetState(s.db, hash) } return common.Hash{} } // GetProof returns the MerkleProof for a given Account -func (self *StateDB) GetProof(a common.Address) ([][]byte, error) { +func (s *StateDB) GetProof(a common.Address) ([][]byte, error) { var proof proofList - err := self.trie.Prove(crypto.Keccak256(a.Bytes()), 0, &proof) + err := s.trie.Prove(crypto.Keccak256(a.Bytes()), 0, &proof) return [][]byte(proof), err } -// GetProof returns the StorageProof for given key -func (self *StateDB) GetStorageProof(a common.Address, key common.Hash) ([][]byte, error) { +// GetStorageProof returns the StorageProof for given key +func (s *StateDB) GetStorageProof(a common.Address, key common.Hash) ([][]byte, error) { var proof proofList - trie := self.StorageTrie(a) + trie := s.StorageTrie(a) if trie == nil { return proof, errors.New("storage trie for requested address does not exist") } @@ -340,32 +362,33 @@ func (self *StateDB) GetStorageProof(a common.Address, key common.Hash) ([][]byt } // GetCommittedState retrieves a value from the given account's committed storage trie. -func (self *StateDB) GetCommittedState(addr common.Address, hash common.Hash) common.Hash { - stateObject := self.getStateObject(addr) +func (s *StateDB) GetCommittedState(addr common.Address, hash common.Hash) common.Hash { + stateObject := s.getStateObject(addr) if stateObject != nil { - return stateObject.GetCommittedState(self.db, hash) + return stateObject.GetCommittedState(s.db, hash) } return common.Hash{} } // Database retrieves the low level database supporting the lower level trie ops. -func (self *StateDB) Database() Database { - return self.db +func (s *StateDB) Database() Database { + return s.db } // StorageTrie returns the storage trie of an account. // The return value is a copy and is nil for non-existent accounts. -func (self *StateDB) StorageTrie(addr common.Address) Trie { - stateObject := self.getStateObject(addr) +func (s *StateDB) StorageTrie(addr common.Address) Trie { + stateObject := s.getStateObject(addr) if stateObject == nil { return nil } - cpy := stateObject.deepCopy(self) - return cpy.updateTrie(self.db) + cpy := stateObject.deepCopy(s) + cpy.updateTrie(s.db) + return cpy.getTrie(s.db) } -func (self *StateDB) HasSuicided(addr common.Address) bool { - stateObject := self.getStateObject(addr) +func (s *StateDB) HasSuicided(addr common.Address) bool { + stateObject := s.getStateObject(addr) if stateObject != nil { return stateObject.suicided } @@ -377,81 +400,81 @@ func (self *StateDB) HasSuicided(addr common.Address) bool { */ // AddBalance adds amount to the account associated with addr. -func (self *StateDB) AddBalance(addr common.Address, amount *big.Int) { - stateObject := self.GetOrNewStateObject(addr) +func (s *StateDB) AddBalance(addr common.Address, amount *big.Int) { + stateObject := s.GetOrNewStateObject(addr) if stateObject != nil { stateObject.AddBalance(amount) } } // SubBalance subtracts amount from the account associated with addr. -func (self *StateDB) SubBalance(addr common.Address, amount *big.Int) { - stateObject := self.GetOrNewStateObject(addr) +func (s *StateDB) SubBalance(addr common.Address, amount *big.Int) { + stateObject := s.GetOrNewStateObject(addr) if stateObject != nil { stateObject.SubBalance(amount) } } -func (self *StateDB) SetBalance(addr common.Address, amount *big.Int) { - stateObject := self.GetOrNewStateObject(addr) +func (s *StateDB) SetBalance(addr common.Address, amount *big.Int) { + stateObject := s.GetOrNewStateObject(addr) if stateObject != nil { stateObject.SetBalance(amount) } } // AddBalance adds amount to the account associated with addr. -func (self *StateDB) AddBalanceMultiCoin(addr common.Address, coinID common.Hash, amount *big.Int) { - stateObject := self.GetOrNewStateObject(addr) +func (s *StateDB) AddBalanceMultiCoin(addr common.Address, coinID common.Hash, amount *big.Int) { + stateObject := s.GetOrNewStateObject(addr) if stateObject != nil { - stateObject.AddBalanceMultiCoin(coinID, amount, self.db) + stateObject.AddBalanceMultiCoin(coinID, amount, s.db) } } // SubBalance subtracts amount from the account associated with addr. -func (self *StateDB) SubBalanceMultiCoin(addr common.Address, coinID common.Hash, amount *big.Int) { - stateObject := self.GetOrNewStateObject(addr) +func (s *StateDB) SubBalanceMultiCoin(addr common.Address, coinID common.Hash, amount *big.Int) { + stateObject := s.GetOrNewStateObject(addr) if stateObject != nil { - stateObject.SubBalanceMultiCoin(coinID, amount, self.db) + stateObject.SubBalanceMultiCoin(coinID, amount, s.db) } } -func (self *StateDB) SetBalanceMultiCoin(addr common.Address, coinID common.Hash, amount *big.Int) { - stateObject := self.GetOrNewStateObject(addr) +func (s *StateDB) SetBalanceMultiCoin(addr common.Address, coinID common.Hash, amount *big.Int) { + stateObject := s.GetOrNewStateObject(addr) if stateObject != nil { - stateObject.SetBalanceMultiCoin(coinID, amount, self.db) + stateObject.SetBalanceMultiCoin(coinID, amount, s.db) } } -func (self *StateDB) SetNonce(addr common.Address, nonce uint64) { - stateObject := self.GetOrNewStateObject(addr) +func (s *StateDB) SetNonce(addr common.Address, nonce uint64) { + stateObject := s.GetOrNewStateObject(addr) if stateObject != nil { stateObject.SetNonce(nonce) } } -func (self *StateDB) SetCode(addr common.Address, code []byte) { - stateObject := self.GetOrNewStateObject(addr) +func (s *StateDB) SetCode(addr common.Address, code []byte) { + stateObject := s.GetOrNewStateObject(addr) if stateObject != nil { stateObject.SetCode(crypto.Keccak256Hash(code), code) } } -func (self *StateDB) SetState(addr common.Address, key, value common.Hash) (res error) { +func (s *StateDB) SetState(addr common.Address, key, value common.Hash) (res error) { res = nil - stateObject := self.GetOrNewStateObject(addr) + stateObject := s.GetOrNewStateObject(addr) if stateObject != nil { if stateObject.data.IsMultiCoin { NormalizeStateKey(&key) } - stateObject.SetState(self.db, key, value) + stateObject.SetState(s.db, key, value) } return } // SetStorage replaces the entire storage for the specified account with given // storage. This function should only be used for debugging. -func (self *StateDB) SetStorage(addr common.Address, storage map[common.Hash]common.Hash) { - stateObject := self.GetOrNewStateObject(addr) +func (s *StateDB) SetStorage(addr common.Address, storage map[common.Hash]common.Hash) { + stateObject := s.GetOrNewStateObject(addr) if stateObject != nil { stateObject.SetStorage(storage) } @@ -462,12 +485,12 @@ func (self *StateDB) SetStorage(addr common.Address, storage map[common.Hash]com // // The account's state object is still available until the state is committed, // getStateObject will return a non-nil account after Suicide. -func (self *StateDB) Suicide(addr common.Address) bool { - stateObject := self.getStateObject(addr) +func (s *StateDB) Suicide(addr common.Address) bool { + stateObject := s.getStateObject(addr) if stateObject == nil { return false } - self.journal.append(suicideChange{ + s.journal.append(suicideChange{ account: &addr, prev: stateObject.suicided, prevbalance: new(big.Int).Set(stateObject.Balance()), @@ -483,90 +506,153 @@ func (self *StateDB) Suicide(addr common.Address) bool { // // updateStateObject writes the given object to the trie. -func (s *StateDB) updateStateObject(stateObject *stateObject) { +func (s *StateDB) updateStateObject(obj *stateObject) { // Track the amount of time wasted on updating the account from the trie if metrics.EnabledExpensive { defer func(start time.Time) { s.AccountUpdates += time.Since(start) }(time.Now()) } // Encode the account and update the account trie - addr := stateObject.Address() + addr := obj.Address() - data, err := rlp.EncodeToBytes(stateObject) + data, err := rlp.EncodeToBytes(obj) if err != nil { panic(fmt.Errorf("can't encode object at %x: %v", addr[:], err)) } - s.setError(s.trie.TryUpdate(addr[:], data)) + if err = s.trie.TryUpdate(addr[:], data); err != nil { + s.setError(fmt.Errorf("updateStateObject (%x) error: %v", addr[:], err)) + } + + // If state snapshotting is active, cache the data til commit. Note, this + // update mechanism is not symmetric to the deletion, because whereas it is + // enough to track account updates at commit time, deletions need tracking + // at transaction boundary level to ensure we capture state clearing. + if s.snap != nil { + s.snapAccounts[obj.addrHash] = snapshot.SlimAccountRLP(obj.data.Nonce, obj.data.Balance, obj.data.Root, obj.data.CodeHash) + } } // deleteStateObject removes the given object from the state trie. -func (s *StateDB) deleteStateObject(stateObject *stateObject) { +func (s *StateDB) deleteStateObject(obj *stateObject) { // Track the amount of time wasted on deleting the account from the trie if metrics.EnabledExpensive { defer func(start time.Time) { s.AccountUpdates += time.Since(start) }(time.Now()) } // Delete the account from the trie - stateObject.deleted = true + addr := obj.Address() + if err := s.trie.TryDelete(addr[:]); err != nil { + s.setError(fmt.Errorf("deleteStateObject (%x) error: %v", addr[:], err)) + } +} - addr := stateObject.Address() - s.setError(s.trie.TryDelete(addr[:])) +// getStateObject retrieves a state object given by the address, returning nil if +// the object is not found or was deleted in this execution context. If you need +// to differentiate between non-existent/just-deleted, use getDeletedStateObject. +func (s *StateDB) getStateObject(addr common.Address) *stateObject { + if obj := s.getDeletedStateObject(addr); obj != nil && !obj.deleted { + return obj + } + return nil } -// Retrieve a state object given by the address. Returns nil if not found. -func (s *StateDB) getStateObject(addr common.Address) (stateObject *stateObject) { - // Prefer live objects +// getDeletedStateObject is similar to getStateObject, but instead of returning +// nil for a deleted state object, it returns the actual object with the deleted +// flag set. This is needed by the state journal to revert to the correct s- +// destructed object instead of wiping all knowledge about the state object. +func (s *StateDB) getDeletedStateObject(addr common.Address) *stateObject { + // Prefer live objects if any is available if obj := s.stateObjects[addr]; obj != nil { - if obj.deleted { - return nil - } return obj } - // Track the amount of time wasted on loading the object from the database - if metrics.EnabledExpensive { - defer func(start time.Time) { s.AccountReads += time.Since(start) }(time.Now()) - } - // Load the object from the database - enc, err := s.trie.TryGet(addr[:]) - if len(enc) == 0 { - s.setError(err) - return nil + // If no live objects are available, attempt to use snapshots + var ( + data *Account + err error + ) + if s.snap != nil { + if metrics.EnabledExpensive { + defer func(start time.Time) { s.SnapshotAccountReads += time.Since(start) }(time.Now()) + } + var acc *snapshot.Account + if acc, err = s.snap.Account(crypto.Keccak256Hash(addr.Bytes())); err == nil { + if acc == nil { + return nil + } + data = &Account{ + Nonce: acc.Nonce, + Balance: acc.Balance, + CodeHash: acc.CodeHash, + Root: common.BytesToHash(acc.Root), + } + if len(data.CodeHash) == 0 { + data.CodeHash = emptyCodeHash + } + if data.Root == (common.Hash{}) { + data.Root = emptyRoot + } + } } - var data Account - if err := rlp.DecodeBytes(enc, &data); err != nil { - log.Error("Failed to decode state object", "addr", addr, "err", err) - return nil + // If snapshot unavailable or reading from it failed, load from the database + if s.snap == nil || err != nil { + if metrics.EnabledExpensive { + defer func(start time.Time) { s.AccountReads += time.Since(start) }(time.Now()) + } + enc, err := s.trie.TryGet(addr.Bytes()) + if err != nil { + s.setError(fmt.Errorf("getDeleteStateObject (%x) error: %v", addr.Bytes(), err)) + return nil + } + if len(enc) == 0 { + return nil + } + data = new(Account) + if err := rlp.DecodeBytes(enc, data); err != nil { + log.Error("Failed to decode state object", "addr", addr, "err", err) + return nil + } } // Insert into the live set - obj := newObject(s, addr, data) + obj := newObject(s, addr, *data) s.setStateObject(obj) return obj } -func (self *StateDB) setStateObject(object *stateObject) { - self.stateObjects[object.Address()] = object +func (s *StateDB) setStateObject(object *stateObject) { + s.stateObjects[object.Address()] = object } -// Retrieve a state object or create a new state object if nil. -func (self *StateDB) GetOrNewStateObject(addr common.Address) *stateObject { - stateObject := self.getStateObject(addr) - if stateObject == nil || stateObject.deleted { - stateObject, _ = self.createObject(addr) +// GetOrNewStateObject retrieves a state object or create a new state object if nil. +func (s *StateDB) GetOrNewStateObject(addr common.Address) *stateObject { + stateObject := s.getStateObject(addr) + if stateObject == nil { + stateObject, _ = s.createObject(addr) } return stateObject } // createObject creates a new state object. If there is an existing account with // the given address, it is overwritten and returned as the second return value. -func (self *StateDB) createObject(addr common.Address) (newobj, prev *stateObject) { - prev = self.getStateObject(addr) - newobj = newObject(self, addr, Account{}) +func (s *StateDB) createObject(addr common.Address) (newobj, prev *stateObject) { + prev = s.getDeletedStateObject(addr) // Note, prev might have been deleted, we need that! + + var prevdestruct bool + if s.snap != nil && prev != nil { + _, prevdestruct = s.snapDestructs[prev.addrHash] + if !prevdestruct { + s.snapDestructs[prev.addrHash] = struct{}{} + } + } + newobj = newObject(s, addr, Account{}) newobj.setNonce(0) // sets the object to dirty if prev == nil { - self.journal.append(createObjectChange{account: &addr}) + s.journal.append(createObjectChange{account: &addr}) } else { - self.journal.append(resetObjectChange{prev: prev}) + s.journal.append(resetObjectChange{prev: prev, prevdestruct: prevdestruct}) + } + s.setStateObject(newobj) + if prev != nil && !prev.deleted { + return newobj, prev } - self.setStateObject(newobj) - return newobj, prev + return newobj, nil } // CreateAccount explicitly creates a state object. If a state object with the address @@ -579,8 +665,8 @@ func (self *StateDB) createObject(addr common.Address) (newobj, prev *stateObjec // 2. tx_create(sha(account ++ nonce)) (note that this gets the address of 1) // // Carrying over the balance ensures that Ether doesn't disappear. -func (self *StateDB) CreateAccount(addr common.Address) { - newObj, prev := self.createObject(addr) +func (s *StateDB) CreateAccount(addr common.Address) { + newObj, prev := s.createObject(addr) if prev != nil { newObj.setBalance(prev.data.Balance) } @@ -617,40 +703,52 @@ func (db *StateDB) ForEachStorage(addr common.Address, cb func(key, value common // Copy creates a deep, independent copy of the state. // Snapshots of the copied state cannot be applied to the copy. -func (self *StateDB) Copy() *StateDB { +func (s *StateDB) Copy() *StateDB { // Copy all the basic fields, initialize the memory ones state := &StateDB{ - db: self.db, - trie: self.db.CopyTrie(self.trie), - stateObjects: make(map[common.Address]*stateObject, len(self.journal.dirties)), - stateObjectsDirty: make(map[common.Address]struct{}, len(self.journal.dirties)), - refund: self.refund, - logs: make(map[common.Hash][]*types.Log, len(self.logs)), - logSize: self.logSize, - preimages: make(map[common.Hash][]byte, len(self.preimages)), - journal: newJournal(), + db: s.db, + trie: s.db.CopyTrie(s.trie), + stateObjects: make(map[common.Address]*stateObject, len(s.journal.dirties)), + stateObjectsPending: make(map[common.Address]struct{}, len(s.stateObjectsPending)), + stateObjectsDirty: make(map[common.Address]struct{}, len(s.journal.dirties)), + refund: s.refund, + logs: make(map[common.Hash][]*types.Log, len(s.logs)), + logSize: s.logSize, + preimages: make(map[common.Hash][]byte, len(s.preimages)), + journal: newJournal(), } // Copy the dirty states, logs, and preimages - for addr := range self.journal.dirties { + for addr := range s.journal.dirties { // As documented [here](https://github.com/ethereum/go-ethereum/pull/16485#issuecomment-380438527), // and in the Finalise-method, there is a case where an object is in the journal but not // in the stateObjects: OOG after touch on ripeMD prior to Byzantium. Thus, we need to check for // nil - if object, exist := self.stateObjects[addr]; exist { + if object, exist := s.stateObjects[addr]; exist { + // Even though the original object is dirty, we are not copying the journal, + // so we need to make sure that anyside effect the journal would have caused + // during a commit (or similar op) is already applied to the copy. state.stateObjects[addr] = object.deepCopy(state) - state.stateObjectsDirty[addr] = struct{}{} + + state.stateObjectsDirty[addr] = struct{}{} // Mark the copy dirty to force internal (code/state) commits + state.stateObjectsPending[addr] = struct{}{} // Mark the copy pending to force external (account) commits } } // Above, we don't copy the actual journal. This means that if the copy is copied, the // loop above will be a no-op, since the copy's journal is empty. // Thus, here we iterate over stateObjects, to enable copies of copies - for addr := range self.stateObjectsDirty { + for addr := range s.stateObjectsPending { + if _, exist := state.stateObjects[addr]; !exist { + state.stateObjects[addr] = s.stateObjects[addr].deepCopy(state) + } + state.stateObjectsPending[addr] = struct{}{} + } + for addr := range s.stateObjectsDirty { if _, exist := state.stateObjects[addr]; !exist { - state.stateObjects[addr] = self.stateObjects[addr].deepCopy(state) - state.stateObjectsDirty[addr] = struct{}{} + state.stateObjects[addr] = s.stateObjects[addr].deepCopy(state) } + state.stateObjectsDirty[addr] = struct{}{} } - for hash, logs := range self.logs { + for hash, logs := range s.logs { cpy := make([]*types.Log, len(logs)) for i, l := range logs { cpy[i] = new(types.Log) @@ -658,46 +756,47 @@ func (self *StateDB) Copy() *StateDB { } state.logs[hash] = cpy } - for hash, preimage := range self.preimages { + for hash, preimage := range s.preimages { state.preimages[hash] = preimage } return state } // Snapshot returns an identifier for the current revision of the state. -func (self *StateDB) Snapshot() int { - id := self.nextRevisionId - self.nextRevisionId++ - self.validRevisions = append(self.validRevisions, revision{id, self.journal.length()}) +func (s *StateDB) Snapshot() int { + id := s.nextRevisionId + s.nextRevisionId++ + s.validRevisions = append(s.validRevisions, revision{id, s.journal.length()}) return id } // RevertToSnapshot reverts all state changes made since the given revision. -func (self *StateDB) RevertToSnapshot(revid int) { +func (s *StateDB) RevertToSnapshot(revid int) { // Find the snapshot in the stack of valid snapshots. - idx := sort.Search(len(self.validRevisions), func(i int) bool { - return self.validRevisions[i].id >= revid + idx := sort.Search(len(s.validRevisions), func(i int) bool { + return s.validRevisions[i].id >= revid }) - if idx == len(self.validRevisions) || self.validRevisions[idx].id != revid { + if idx == len(s.validRevisions) || s.validRevisions[idx].id != revid { panic(fmt.Errorf("revision id %v cannot be reverted", revid)) } - snapshot := self.validRevisions[idx].journalIndex + snapshot := s.validRevisions[idx].journalIndex // Replay the journal to undo changes and remove invalidated snapshots - self.journal.revert(self, snapshot) - self.validRevisions = self.validRevisions[:idx] + s.journal.revert(s, snapshot) + s.validRevisions = s.validRevisions[:idx] } // GetRefund returns the current value of the refund counter. -func (self *StateDB) GetRefund() uint64 { - return self.refund +func (s *StateDB) GetRefund() uint64 { + return s.refund } -// Finalise finalises the state by removing the self destructed objects -// and clears the journal as well as the refunds. +// Finalise finalises the state by removing the s destructed objects and clears +// the journal as well as the refunds. Finalise, however, will not push any updates +// into the tries just yet. Only IntermediateRoot or Commit will do that. func (s *StateDB) Finalise(deleteEmptyObjects bool) { for addr := range s.journal.dirties { - stateObject, exist := s.stateObjects[addr] + obj, exist := s.stateObjects[addr] if !exist { // ripeMD is 'touched' at block 1714175, in tx 0x1237f737031e40bcde4a8b7e717b2d15e3ecadfe49bb1bbc71ee9deb09c6fcf2 // That tx goes out of gas, and although the notion of 'touched' does not exist there, the @@ -707,13 +806,22 @@ func (s *StateDB) Finalise(deleteEmptyObjects bool) { // Thus, we can safely ignore it here continue } - - if stateObject.suicided || (deleteEmptyObjects && stateObject.empty()) { - s.deleteStateObject(stateObject) + if obj.suicided || (deleteEmptyObjects && obj.empty()) { + obj.deleted = true + + // If state snapshotting is active, also mark the destruction there. + // Note, we can't do this only at the end of a block because multiple + // transactions within the same block might self destruct and then + // ressurrect an account; but the snapshotter needs both events. + if s.snap != nil { + s.snapDestructs[obj.addrHash] = struct{}{} // We need to maintain account deletions explicitly (will remain set indefinitely) + delete(s.snapAccounts, obj.addrHash) // Clear out any previously updated account data (may be recreated via a ressurrect) + delete(s.snapStorage, obj.addrHash) // Clear out any previously updated storage data (may be recreated via a ressurrect) + } } else { - stateObject.updateRoot(s.db) - s.updateStateObject(stateObject) + obj.finalise() } + s.stateObjectsPending[addr] = struct{}{} s.stateObjectsDirty[addr] = struct{}{} } // Invalidate journal because reverting across transactions is not allowed. @@ -724,8 +832,21 @@ func (s *StateDB) Finalise(deleteEmptyObjects bool) { // It is called in between transactions to get the root hash that // goes into transaction receipts. func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { + // Finalise all the dirty storage states and write them into the tries s.Finalise(deleteEmptyObjects) + for addr := range s.stateObjectsPending { + obj := s.stateObjects[addr] + if obj.deleted { + s.deleteStateObject(obj) + } else { + obj.updateRoot(s.db) + s.updateStateObject(obj) + } + } + if len(s.stateObjectsPending) > 0 { + s.stateObjectsPending = make(map[common.Address]struct{}) + } // Track the amount of time wasted on hashing the account trie if metrics.EnabledExpensive { defer func(start time.Time) { s.AccountHashes += time.Since(start) }(time.Now()) @@ -735,65 +856,86 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { // Prepare sets the current transaction hash and index and block hash which is // used when the EVM emits new state logs. -func (self *StateDB) Prepare(thash, bhash common.Hash, ti int) { - self.thash = thash - self.bhash = bhash - self.txIndex = ti +func (s *StateDB) Prepare(thash, bhash common.Hash, ti int) { + s.thash = thash + s.bhash = bhash + s.txIndex = ti } func (s *StateDB) clearJournalAndRefund() { - s.journal = newJournal() - s.validRevisions = s.validRevisions[:0] - s.refund = 0 + if len(s.journal.entries) > 0 { + s.journal = newJournal() + s.refund = 0 + } + s.validRevisions = s.validRevisions[:0] // Snapshots can be created without journal entires } // Commit writes the state to the underlying in-memory trie database. -func (s *StateDB) Commit(deleteEmptyObjects bool) (root common.Hash, err error) { - defer s.clearJournalAndRefund() - - for addr := range s.journal.dirties { - s.stateObjectsDirty[addr] = struct{}{} +func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { + if s.dbErr != nil { + return common.Hash{}, fmt.Errorf("commit aborted due to earlier error: %v", s.dbErr) } + // Finalize any pending changes and merge everything into the tries + s.IntermediateRoot(deleteEmptyObjects) + // Commit objects to the trie, measuring the elapsed time - for addr, stateObject := range s.stateObjects { - _, isDirty := s.stateObjectsDirty[addr] - switch { - case stateObject.suicided || (isDirty && deleteEmptyObjects && stateObject.empty()): - // If the object has been removed, don't bother syncing it - // and just mark it for deletion in the trie. - s.deleteStateObject(stateObject) - case isDirty: + codeWriter := s.db.TrieDB().DiskDB().NewBatch() + for addr := range s.stateObjectsDirty { + if obj := s.stateObjects[addr]; !obj.deleted { // Write any contract code associated with the state object - if stateObject.code != nil && stateObject.dirtyCode { - s.db.TrieDB().InsertBlob(common.BytesToHash(stateObject.CodeHash()), stateObject.code) - stateObject.dirtyCode = false + if obj.code != nil && obj.dirtyCode { + rawdb.WriteCode(codeWriter, common.BytesToHash(obj.CodeHash()), obj.code) + obj.dirtyCode = false } - // Write any storage changes in the state object to its storage trie. - if err := stateObject.CommitTrie(s.db); err != nil { + // Write any storage changes in the state object to its storage trie + if err := obj.CommitTrie(s.db); err != nil { return common.Hash{}, err } - // Update the object in the main account trie. - s.updateStateObject(stateObject) } - delete(s.stateObjectsDirty, addr) + } + if len(s.stateObjectsDirty) > 0 { + s.stateObjectsDirty = make(map[common.Address]struct{}) + } + if codeWriter.ValueSize() > 0 { + if err := codeWriter.Write(); err != nil { + log.Crit("Failed to commit dirty codes", "error", err) + } } // Write the account trie changes, measuing the amount of wasted time + var start time.Time if metrics.EnabledExpensive { - defer func(start time.Time) { s.AccountCommits += time.Since(start) }(time.Now()) + start = time.Now() } - root, err = s.trie.Commit(func(leaf []byte, parent common.Hash) error { - var account Account + // The onleaf func is called _serially_, so we can reuse the same account + // for unmarshalling every time. + var account Account + root, err := s.trie.Commit(func(path []byte, leaf []byte, parent common.Hash) error { if err := rlp.DecodeBytes(leaf, &account); err != nil { return nil } if account.Root != emptyRoot { s.db.TrieDB().Reference(account.Root, parent) } - code := common.BytesToHash(account.CodeHash) - if code != emptyCode { - s.db.TrieDB().Reference(code, parent) - } return nil }) + if metrics.EnabledExpensive { + s.AccountCommits += time.Since(start) + } + // If snapshotting is enabled, update the snapshot tree with this new version + if s.snap != nil { + if metrics.EnabledExpensive { + defer func(start time.Time) { s.SnapshotCommits += time.Since(start) }(time.Now()) + } + // Only update if there's a state transition (skip empty Clique blocks) + if parent := s.snap.Root(); parent != root { + if err := s.snaps.Update(root, parent, s.snapDestructs, s.snapAccounts, s.snapStorage); err != nil { + log.Warn("Failed to update snapshot tree", "from", parent, "to", root, "err", err) + } + if err := s.snaps.Cap(root, 127); err != nil { // Persistent layer is 128th, the last available trie + log.Warn("Failed to cap snapshot tree", "root", root, "layers", 127, "err", err) + } + } + s.snap, s.snapDestructs, s.snapAccounts, s.snapStorage = nil, nil, nil, nil + } return root, err } diff --git a/core/state_processor.go b/core/state_processor.go index ab8759a..46ecd48 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -23,8 +23,8 @@ import ( "github.com/ava-labs/coreth/core/types" "github.com/ava-labs/coreth/core/vm" "github.com/ava-labs/coreth/params" - "github.com/ava-labs/go-ethereum/common" - "github.com/ava-labs/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" ) // StateProcessor is a basic Processor, which takes care of transitioning @@ -68,7 +68,7 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg // Iterate over and process the individual transactions for i, tx := range block.Transactions() { statedb.Prepare(tx.Hash(), block.Hash(), i) - receipt, _, err := ApplyTransaction(p.config, p.bc, nil, gp, statedb, header, tx, usedGas, cfg) + receipt, err := ApplyTransaction(p.config, p.bc, nil, gp, statedb, header, tx, usedGas, cfg) if err != nil { return nil, nil, 0, err } @@ -88,10 +88,10 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg // and uses the input parameters for its environment. It returns the receipt // for the transaction, gas used and an error if the transaction failed, // indicating the block was invalid. -func ApplyTransaction(config *params.ChainConfig, bc ChainContext, author *common.Address, gp *GasPool, statedb *state.StateDB, header *types.Header, tx *types.Transaction, usedGas *uint64, cfg vm.Config) (*types.Receipt, uint64, error) { +func ApplyTransaction(config *params.ChainConfig, bc ChainContext, author *common.Address, gp *GasPool, statedb *state.StateDB, header *types.Header, tx *types.Transaction, usedGas *uint64, cfg vm.Config) (*types.Receipt, error) { msg, err := tx.AsMessage(types.MakeSigner(config, header.Number)) if err != nil { - return nil, 0, err + return nil, err } // Create a new context to be used in the EVM environment context := NewEVMContext(msg, header, bc, author) @@ -99,9 +99,9 @@ func ApplyTransaction(config *params.ChainConfig, bc ChainContext, author *commo // about the transaction and calling mechanisms. vmenv := vm.NewEVM(context, statedb, config, cfg) // Apply the transaction to the current state (included in the env) - _, gas, failed, err := ApplyMessage(vmenv, msg, gp) + result, err := ApplyMessage(vmenv, msg, gp) if err != nil { - return nil, 0, err + return nil, err } // Update the state with pending changes var root []byte @@ -110,13 +110,13 @@ func ApplyTransaction(config *params.ChainConfig, bc ChainContext, author *commo } else { root = statedb.IntermediateRoot(config.IsEIP158(header.Number)).Bytes() } - *usedGas += gas + *usedGas += result.UsedGas // Create a new receipt for the transaction, storing the intermediate root and gas used by the tx // based on the eip phase, we're passing whether the root touch-delete accounts. - receipt := types.NewReceipt(root, failed, *usedGas) + receipt := types.NewReceipt(root, result.Failed(), *usedGas) receipt.TxHash = tx.Hash() - receipt.GasUsed = gas + receipt.GasUsed = result.UsedGas // if the transaction created a contract, store the creation address in the receipt. if msg.To() == nil { receipt.ContractAddress = crypto.CreateAddress(vmenv.Context.Origin, tx.Nonce()) @@ -128,5 +128,5 @@ func ApplyTransaction(config *params.ChainConfig, bc ChainContext, author *commo receipt.BlockNumber = header.Number receipt.TransactionIndex = uint(statedb.TxIndex()) - return receipt, gas, err + return receipt, err } diff --git a/hacked-list.txt b/hacked-list.txt new file mode 100644 index 0000000..92d0517 --- /dev/null +++ b/hacked-list.txt @@ -0,0 +1,44 @@ +./consensus/dummy/consensus.go +./consensus/clique/clique.go +./consensus/consensus.go +./consensus/ethash/consensus.go +./core/blockchain.go +./core/events.go +./core/evm.go +./core/genesis.go +./core/gen_genesis_account.go +./core/gen_genesis.go +./core/rawdb/accessors_chain.go +./core/state/journal.go +./core/state_processor.go +./core/state/statedb.go +./core/state/state_object.go + +./core/tx_pool.go +./core/types/block.go +./core/vm/errors.go +./core/vm/evm.go +./core/vm/instructions.go +./core/vm/interface.go +./core/vm/interpreter.go +./core/vm/jump_table.go +./core/vm/memory_table.go +./core/vm/opcodes.go +./eth/api_backend.go +./eth/api.go +./eth/api_tracer.go +./eth/backend.go +./eth/config.go +./eth/gen_config.go +./eth/tracers/internal/tracers/assets.go +./internal/ethapi/api.go +./internal/ethapi/backend.go +./miner/miner.go +./miner/worker.go +./node/api.go +./node/config.go +./node/node.go +./node/service.go +./params/protocol_params.go +./rpc/client.go +./rpc/types.go -- cgit v1.2.3-70-g09d2 From b1ac5e6ce73c37378e575d6291e3c5f1ee170430 Mon Sep 17 00:00:00 2001 From: Determinant Date: Wed, 16 Sep 2020 19:00:25 -0400 Subject: build success --- consensus/clique/clique.go | 2 +- consensus/dummy/consensus.go | 31 +- consensus/ethash/consensus.go | 2 +- core/blockchain.go | 2 +- core/forkid/forkid.go | 258 ++++++++ core/rawdb/accessors_state.go | 96 +++ core/rawdb/chain_iterator.go | 304 ++++++++++ core/state/snapshot/difflayer_test.go | 400 ------------- core/state/snapshot/disklayer_test.go | 511 ---------------- core/state/snapshot/iterator_test.go | 1046 --------------------------------- core/state/snapshot/snapshot_test.go | 371 ------------ core/state/snapshot/wipe_test.go | 124 ---- core/vm/instructions.go | 14 +- coreth.go | 19 +- eth/api.go | 2 +- eth/backend.go | 10 +- eth/protocol.go | 1 + internal/ethapi/api.go | 1 - internal/ethapi/backend.go | 1 + miner/miner.go | 3 +- node/api.go | 1 + node/node.go | 34 +- rpc/metrics.go | 39 ++ 23 files changed, 777 insertions(+), 2495 deletions(-) create mode 100644 core/forkid/forkid.go create mode 100644 core/rawdb/accessors_state.go create mode 100644 core/rawdb/chain_iterator.go delete mode 100644 core/state/snapshot/difflayer_test.go delete mode 100644 core/state/snapshot/disklayer_test.go delete mode 100644 core/state/snapshot/iterator_test.go delete mode 100644 core/state/snapshot/snapshot_test.go delete mode 100644 core/state/snapshot/wipe_test.go create mode 100644 rpc/metrics.go (limited to 'core/rawdb') diff --git a/consensus/clique/clique.go b/consensus/clique/clique.go index d239aca..b27525b 100644 --- a/consensus/clique/clique.go +++ b/consensus/clique/clique.go @@ -566,7 +566,7 @@ func (c *Clique) FinalizeAndAssemble(chain consensus.ChainHeaderReader, header * header.UncleHash = types.CalcUncleHash(nil) // Assemble and return the final block for sealing - return types.NewBlock(header, txs, nil, receipts, new(trie.Trie)), nil + return types.NewBlock(header, txs, nil, receipts, new(trie.Trie), nil), nil } // Authorize injects a private key into the consensus engine to mint new blocks diff --git a/consensus/dummy/consensus.go b/consensus/dummy/consensus.go index 2108684..da63673 100644 --- a/consensus/dummy/consensus.go +++ b/consensus/dummy/consensus.go @@ -13,14 +13,15 @@ import ( "github.com/ava-labs/coreth/core/types" "github.com/ava-labs/coreth/params" "github.com/ava-labs/coreth/rpc" + mapset "github.com/deckarep/golang-set" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/rlp" - mapset "github.com/deckarep/golang-set" + "github.com/ethereum/go-ethereum/trie" ) -type OnFinalizeCallbackType = func(chain consensus.ChainReader, header *types.Header, state *state.StateDB, txs []*types.Transaction, uncles []*types.Header) +type OnFinalizeCallbackType = func(chain consensus.ChainHeaderReader, header *types.Header, state *state.StateDB, txs []*types.Transaction, uncles []*types.Header) type OnFinalizeAndAssembleCallbackType = func(state *state.StateDB, txs []*types.Transaction) ([]byte, error) -type OnAPIsCallbackType = func(consensus.ChainReader) []rpc.API +type OnAPIsCallbackType = func(consensus.ChainHeaderReader) []rpc.API type OnExtraStateChangeType = func(block *types.Block, statedb *state.StateDB) error type ConsensusCallbacks struct { @@ -55,7 +56,7 @@ var ( ) // modified from consensus.go -func (self *DummyEngine) verifyHeader(chain consensus.ChainReader, header, parent *types.Header, uncle bool, seal bool) error { +func (self *DummyEngine) verifyHeader(chain consensus.ChainHeaderReader, header, parent *types.Header, uncle bool, seal bool) error { // Ensure that the header's extra-data section is of a reasonable size if uint64(len(header.Extra)) > params.MaximumExtraDataSize { return fmt.Errorf("extra-data too long: %d > %d", len(header.Extra), params.MaximumExtraDataSize) @@ -103,7 +104,7 @@ func (self *DummyEngine) verifyHeader(chain consensus.ChainReader, header, paren return nil } -func (self *DummyEngine) verifyHeaderWorker(chain consensus.ChainReader, headers []*types.Header, seals []bool, index int) error { +func (self *DummyEngine) verifyHeaderWorker(chain consensus.ChainHeaderReader, headers []*types.Header, seals []bool, index int) error { var parent *types.Header if index == 0 { parent = chain.GetHeader(headers[0].ParentHash, headers[0].Number.Uint64()-1) @@ -123,7 +124,7 @@ func (self *DummyEngine) Author(header *types.Header) (common.Address, error) { return header.Coinbase, nil } -func (self *DummyEngine) VerifyHeader(chain consensus.ChainReader, header *types.Header, seal bool) error { +func (self *DummyEngine) VerifyHeader(chain consensus.ChainHeaderReader, header *types.Header, seal bool) error { // Short circuit if the header is known, or it's parent not number := header.Number.Uint64() if chain.GetHeader(header.Hash(), number) != nil { @@ -137,7 +138,7 @@ func (self *DummyEngine) VerifyHeader(chain consensus.ChainReader, header *types return self.verifyHeader(chain, header, parent, false, seal) } -func (self *DummyEngine) VerifyHeaders(chain consensus.ChainReader, headers []*types.Header, seals []bool) (chan<- struct{}, <-chan error) { +func (self *DummyEngine) VerifyHeaders(chain consensus.ChainHeaderReader, headers []*types.Header, seals []bool) (chan<- struct{}, <-chan error) { // Spawn as many workers as allowed threads workers := runtime.GOMAXPROCS(0) if len(headers) < workers { @@ -239,17 +240,17 @@ func (self *DummyEngine) VerifyUncles(chain consensus.ChainReader, block *types. return nil } -func (self *DummyEngine) VerifySeal(chain consensus.ChainReader, header *types.Header) error { +func (self *DummyEngine) VerifySeal(chain consensus.ChainHeaderReader, header *types.Header) error { return nil } -func (self *DummyEngine) Prepare(chain consensus.ChainReader, header *types.Header) error { +func (self *DummyEngine) Prepare(chain consensus.ChainHeaderReader, header *types.Header) error { header.Difficulty = big.NewInt(1) return nil } func (self *DummyEngine) Finalize( - chain consensus.ChainReader, header *types.Header, + chain consensus.ChainHeaderReader, header *types.Header, state *state.StateDB, txs []*types.Transaction, uncles []*types.Header) { if self.cb.OnFinalize != nil { @@ -259,7 +260,7 @@ func (self *DummyEngine) Finalize( header.Root = state.IntermediateRoot(chain.Config().IsEIP158(header.Number)) } -func (self *DummyEngine) FinalizeAndAssemble(chain consensus.ChainReader, header *types.Header, state *state.StateDB, txs []*types.Transaction, +func (self *DummyEngine) FinalizeAndAssemble(chain consensus.ChainHeaderReader, header *types.Header, state *state.StateDB, txs []*types.Transaction, uncles []*types.Header, receipts []*types.Receipt) (*types.Block, error) { var extdata []byte if self.cb.OnFinalizeAndAssemble != nil { @@ -273,10 +274,10 @@ func (self *DummyEngine) FinalizeAndAssemble(chain consensus.ChainReader, header header.Root = state.IntermediateRoot(chain.Config().IsEIP158(header.Number)) // Header seems complete, assemble into a block and return - return types.NewBlock(header, txs, uncles, receipts, extdata), nil + return types.NewBlock(header, txs, uncles, receipts, new(trie.Trie), extdata), nil } -func (self *DummyEngine) Seal(chain consensus.ChainReader, block *types.Block, results chan<- *types.Block, stop <-chan struct{}) (err error) { +func (self *DummyEngine) Seal(chain consensus.ChainHeaderReader, block *types.Block, results chan<- *types.Block, stop <-chan struct{}) (err error) { if self.cb.OnSeal != nil { err = self.cb.OnSeal(block) } else { @@ -314,11 +315,11 @@ func (self *DummyEngine) SealHash(header *types.Header) (hash common.Hash) { return hash } -func (self *DummyEngine) CalcDifficulty(chain consensus.ChainReader, time uint64, parent *types.Header) *big.Int { +func (self *DummyEngine) CalcDifficulty(chain consensus.ChainHeaderReader, time uint64, parent *types.Header) *big.Int { return big.NewInt(1) } -func (self *DummyEngine) APIs(chain consensus.ChainReader) (res []rpc.API) { +func (self *DummyEngine) APIs(chain consensus.ChainHeaderReader) (res []rpc.API) { res = nil if self.cb.OnAPIs != nil { res = self.cb.OnAPIs(chain) diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index 151761c..dc56b6f 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -584,7 +584,7 @@ func (ethash *Ethash) FinalizeAndAssemble(chain consensus.ChainHeaderReader, hea header.Root = state.IntermediateRoot(chain.Config().IsEIP158(header.Number)) // Header seems complete, assemble into a block and return - return types.NewBlock(header, txs, uncles, receipts, new(trie.Trie)), nil + return types.NewBlock(header, txs, uncles, receipts, new(trie.Trie), nil), nil } // SealHash returns the hash of a block prior to it being sealed. diff --git a/core/blockchain.go b/core/blockchain.go index b861220..82e3b6c 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -2498,6 +2498,6 @@ func (bc *BlockChain) ManualHead(hash common.Hash) error { } bc.chainmu.Lock() defer bc.chainmu.Unlock() - bc.insert(block) + bc.writeHeadBlock(block) return nil } diff --git a/core/forkid/forkid.go b/core/forkid/forkid.go new file mode 100644 index 0000000..1d6563d --- /dev/null +++ b/core/forkid/forkid.go @@ -0,0 +1,258 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package forkid implements EIP-2124 (https://eips.ethereum.org/EIPS/eip-2124). +package forkid + +import ( + "encoding/binary" + "errors" + "hash/crc32" + "math" + "math/big" + "reflect" + "strings" + + "github.com/ava-labs/coreth/core/types" + "github.com/ava-labs/coreth/params" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" +) + +var ( + // ErrRemoteStale is returned by the validator if a remote fork checksum is a + // subset of our already applied forks, but the announced next fork block is + // not on our already passed chain. + ErrRemoteStale = errors.New("remote needs update") + + // ErrLocalIncompatibleOrStale is returned by the validator if a remote fork + // checksum does not match any local checksum variation, signalling that the + // two chains have diverged in the past at some point (possibly at genesis). + ErrLocalIncompatibleOrStale = errors.New("local incompatible or needs update") +) + +// Blockchain defines all necessary method to build a forkID. +type Blockchain interface { + // Config retrieves the chain's fork configuration. + Config() *params.ChainConfig + + // Genesis retrieves the chain's genesis block. + Genesis() *types.Block + + // CurrentHeader retrieves the current head header of the canonical chain. + CurrentHeader() *types.Header +} + +// ID is a fork identifier as defined by EIP-2124. +type ID struct { + Hash [4]byte // CRC32 checksum of the genesis block and passed fork block numbers + Next uint64 // Block number of the next upcoming fork, or 0 if no forks are known +} + +// Filter is a fork id filter to validate a remotely advertised ID. +type Filter func(id ID) error + +// NewID calculates the Ethereum fork ID from the chain config and head. +func NewID(chain Blockchain) ID { + return newID( + chain.Config(), + chain.Genesis().Hash(), + chain.CurrentHeader().Number.Uint64(), + ) +} + +// newID is the internal version of NewID, which takes extracted values as its +// arguments instead of a chain. The reason is to allow testing the IDs without +// having to simulate an entire blockchain. +func newID(config *params.ChainConfig, genesis common.Hash, head uint64) ID { + // Calculate the starting checksum from the genesis hash + hash := crc32.ChecksumIEEE(genesis[:]) + + // Calculate the current fork checksum and the next fork block + var next uint64 + for _, fork := range gatherForks(config) { + if fork <= head { + // Fork already passed, checksum the previous hash and the fork number + hash = checksumUpdate(hash, fork) + continue + } + next = fork + break + } + return ID{Hash: checksumToBytes(hash), Next: next} +} + +// NewFilter creates a filter that returns if a fork ID should be rejected or not +// based on the local chain's status. +func NewFilter(chain Blockchain) Filter { + return newFilter( + chain.Config(), + chain.Genesis().Hash(), + func() uint64 { + return chain.CurrentHeader().Number.Uint64() + }, + ) +} + +// NewStaticFilter creates a filter at block zero. +func NewStaticFilter(config *params.ChainConfig, genesis common.Hash) Filter { + head := func() uint64 { return 0 } + return newFilter(config, genesis, head) +} + +// newFilter is the internal version of NewFilter, taking closures as its arguments +// instead of a chain. The reason is to allow testing it without having to simulate +// an entire blockchain. +func newFilter(config *params.ChainConfig, genesis common.Hash, headfn func() uint64) Filter { + // Calculate the all the valid fork hash and fork next combos + var ( + forks = gatherForks(config) + sums = make([][4]byte, len(forks)+1) // 0th is the genesis + ) + hash := crc32.ChecksumIEEE(genesis[:]) + sums[0] = checksumToBytes(hash) + for i, fork := range forks { + hash = checksumUpdate(hash, fork) + sums[i+1] = checksumToBytes(hash) + } + // Add two sentries to simplify the fork checks and don't require special + // casing the last one. + forks = append(forks, math.MaxUint64) // Last fork will never be passed + + // Create a validator that will filter out incompatible chains + return func(id ID) error { + // Run the fork checksum validation ruleset: + // 1. If local and remote FORK_CSUM matches, compare local head to FORK_NEXT. + // The two nodes are in the same fork state currently. They might know + // of differing future forks, but that's not relevant until the fork + // triggers (might be postponed, nodes might be updated to match). + // 1a. A remotely announced but remotely not passed block is already passed + // locally, disconnect, since the chains are incompatible. + // 1b. No remotely announced fork; or not yet passed locally, connect. + // 2. If the remote FORK_CSUM is a subset of the local past forks and the + // remote FORK_NEXT matches with the locally following fork block number, + // connect. + // Remote node is currently syncing. It might eventually diverge from + // us, but at this current point in time we don't have enough information. + // 3. If the remote FORK_CSUM is a superset of the local past forks and can + // be completed with locally known future forks, connect. + // Local node is currently syncing. It might eventually diverge from + // the remote, but at this current point in time we don't have enough + // information. + // 4. Reject in all other cases. + head := headfn() + for i, fork := range forks { + // If our head is beyond this fork, continue to the next (we have a dummy + // fork of maxuint64 as the last item to always fail this check eventually). + if head > fork { + continue + } + // Found the first unpassed fork block, check if our current state matches + // the remote checksum (rule #1). + if sums[i] == id.Hash { + // Fork checksum matched, check if a remote future fork block already passed + // locally without the local node being aware of it (rule #1a). + if id.Next > 0 && head >= id.Next { + return ErrLocalIncompatibleOrStale + } + // Haven't passed locally a remote-only fork, accept the connection (rule #1b). + return nil + } + // The local and remote nodes are in different forks currently, check if the + // remote checksum is a subset of our local forks (rule #2). + for j := 0; j < i; j++ { + if sums[j] == id.Hash { + // Remote checksum is a subset, validate based on the announced next fork + if forks[j] != id.Next { + return ErrRemoteStale + } + return nil + } + } + // Remote chain is not a subset of our local one, check if it's a superset by + // any chance, signalling that we're simply out of sync (rule #3). + for j := i + 1; j < len(sums); j++ { + if sums[j] == id.Hash { + // Yay, remote checksum is a superset, ignore upcoming forks + return nil + } + } + // No exact, subset or superset match. We are on differing chains, reject. + return ErrLocalIncompatibleOrStale + } + log.Error("Impossible fork ID validation", "id", id) + return nil // Something's very wrong, accept rather than reject + } +} + +// checksumUpdate calculates the next IEEE CRC32 checksum based on the previous +// one and a fork block number (equivalent to CRC32(original-blob || fork)). +func checksumUpdate(hash uint32, fork uint64) uint32 { + var blob [8]byte + binary.BigEndian.PutUint64(blob[:], fork) + return crc32.Update(hash, crc32.IEEETable, blob[:]) +} + +// checksumToBytes converts a uint32 checksum into a [4]byte array. +func checksumToBytes(hash uint32) [4]byte { + var blob [4]byte + binary.BigEndian.PutUint32(blob[:], hash) + return blob +} + +// gatherForks gathers all the known forks and creates a sorted list out of them. +func gatherForks(config *params.ChainConfig) []uint64 { + // Gather all the fork block numbers via reflection + kind := reflect.TypeOf(params.ChainConfig{}) + conf := reflect.ValueOf(config).Elem() + + var forks []uint64 + for i := 0; i < kind.NumField(); i++ { + // Fetch the next field and skip non-fork rules + field := kind.Field(i) + if !strings.HasSuffix(field.Name, "Block") { + continue + } + if field.Type != reflect.TypeOf(new(big.Int)) { + continue + } + // Extract the fork rule block number and aggregate it + rule := conf.Field(i).Interface().(*big.Int) + if rule != nil { + forks = append(forks, rule.Uint64()) + } + } + // Sort the fork block numbers to permit chronological XOR + for i := 0; i < len(forks); i++ { + for j := i + 1; j < len(forks); j++ { + if forks[i] > forks[j] { + forks[i], forks[j] = forks[j], forks[i] + } + } + } + // Deduplicate block numbers applying multiple forks + for i := 1; i < len(forks); i++ { + if forks[i] == forks[i-1] { + forks = append(forks[:i], forks[i+1:]...) + i-- + } + } + // Skip any forks in block 0, that's the genesis ruleset + if len(forks) > 0 && forks[0] == 0 { + forks = forks[1:] + } + return forks +} diff --git a/core/rawdb/accessors_state.go b/core/rawdb/accessors_state.go new file mode 100644 index 0000000..6112de0 --- /dev/null +++ b/core/rawdb/accessors_state.go @@ -0,0 +1,96 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rawdb + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" +) + +// ReadPreimage retrieves a single preimage of the provided hash. +func ReadPreimage(db ethdb.KeyValueReader, hash common.Hash) []byte { + data, _ := db.Get(preimageKey(hash)) + return data +} + +// WritePreimages writes the provided set of preimages to the database. +func WritePreimages(db ethdb.KeyValueWriter, preimages map[common.Hash][]byte) { + for hash, preimage := range preimages { + if err := db.Put(preimageKey(hash), preimage); err != nil { + log.Crit("Failed to store trie preimage", "err", err) + } + } + preimageCounter.Inc(int64(len(preimages))) + preimageHitCounter.Inc(int64(len(preimages))) +} + +// ReadCode retrieves the contract code of the provided code hash. +func ReadCode(db ethdb.KeyValueReader, hash common.Hash) []byte { + // Try with the legacy code scheme first, if not then try with current + // scheme. Since most of the code will be found with legacy scheme. + // + // todo(rjl493456442) change the order when we forcibly upgrade the code + // scheme with snapshot. + data, _ := db.Get(hash[:]) + if len(data) != 0 { + return data + } + return ReadCodeWithPrefix(db, hash) +} + +// ReadCodeWithPrefix retrieves the contract code of the provided code hash. +// The main difference between this function and ReadCode is this function +// will only check the existence with latest scheme(with prefix). +func ReadCodeWithPrefix(db ethdb.KeyValueReader, hash common.Hash) []byte { + data, _ := db.Get(codeKey(hash)) + return data +} + +// WriteCode writes the provided contract code database. +func WriteCode(db ethdb.KeyValueWriter, hash common.Hash, code []byte) { + if err := db.Put(codeKey(hash), code); err != nil { + log.Crit("Failed to store contract code", "err", err) + } +} + +// DeleteCode deletes the specified contract code from the database. +func DeleteCode(db ethdb.KeyValueWriter, hash common.Hash) { + if err := db.Delete(codeKey(hash)); err != nil { + log.Crit("Failed to delete contract code", "err", err) + } +} + +// ReadTrieNode retrieves the trie node of the provided hash. +func ReadTrieNode(db ethdb.KeyValueReader, hash common.Hash) []byte { + data, _ := db.Get(hash.Bytes()) + return data +} + +// WriteTrieNode writes the provided trie node database. +func WriteTrieNode(db ethdb.KeyValueWriter, hash common.Hash, node []byte) { + if err := db.Put(hash.Bytes(), node); err != nil { + log.Crit("Failed to store trie node", "err", err) + } +} + +// DeleteTrieNode deletes the specified trie node from the database. +func DeleteTrieNode(db ethdb.KeyValueWriter, hash common.Hash) { + if err := db.Delete(hash.Bytes()); err != nil { + log.Crit("Failed to delete trie node", "err", err) + } +} diff --git a/core/rawdb/chain_iterator.go b/core/rawdb/chain_iterator.go new file mode 100644 index 0000000..3130e92 --- /dev/null +++ b/core/rawdb/chain_iterator.go @@ -0,0 +1,304 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rawdb + +import ( + "runtime" + "sync/atomic" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/prque" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" + "golang.org/x/crypto/sha3" +) + +// InitDatabaseFromFreezer reinitializes an empty database from a previous batch +// of frozen ancient blocks. The method iterates over all the frozen blocks and +// injects into the database the block hash->number mappings. +func InitDatabaseFromFreezer(db ethdb.Database) { + // If we can't access the freezer or it's empty, abort + frozen, err := db.Ancients() + if err != nil || frozen == 0 { + return + } + var ( + batch = db.NewBatch() + start = time.Now() + logged = start.Add(-7 * time.Second) // Unindex during import is fast, don't double log + hash common.Hash + ) + for i := uint64(0); i < frozen; i++ { + // Since the freezer has all data in sequential order on a file, + // it would be 'neat' to read more data in one go, and let the + // freezerdb return N items (e.g up to 1000 items per go) + // That would require an API change in Ancients though + if h, err := db.Ancient(freezerHashTable, i); err != nil { + log.Crit("Failed to init database from freezer", "err", err) + } else { + hash = common.BytesToHash(h) + } + WriteHeaderNumber(batch, hash, i) + // If enough data was accumulated in memory or we're at the last block, dump to disk + if batch.ValueSize() > ethdb.IdealBatchSize { + if err := batch.Write(); err != nil { + log.Crit("Failed to write data to db", "err", err) + } + batch.Reset() + } + // If we've spent too much time already, notify the user of what we're doing + if time.Since(logged) > 8*time.Second { + log.Info("Initializing database from freezer", "total", frozen, "number", i, "hash", hash, "elapsed", common.PrettyDuration(time.Since(start))) + logged = time.Now() + } + } + if err := batch.Write(); err != nil { + log.Crit("Failed to write data to db", "err", err) + } + batch.Reset() + + WriteHeadHeaderHash(db, hash) + WriteHeadFastBlockHash(db, hash) + log.Info("Initialized database from freezer", "blocks", frozen, "elapsed", common.PrettyDuration(time.Since(start))) +} + +type blockTxHashes struct { + number uint64 + hashes []common.Hash +} + +// iterateTransactions iterates over all transactions in the (canon) block +// number(s) given, and yields the hashes on a channel +func iterateTransactions(db ethdb.Database, from uint64, to uint64, reverse bool) (chan *blockTxHashes, chan struct{}) { + // One thread sequentially reads data from db + type numberRlp struct { + number uint64 + rlp rlp.RawValue + } + if to == from { + return nil, nil + } + threads := to - from + if cpus := runtime.NumCPU(); threads > uint64(cpus) { + threads = uint64(cpus) + } + var ( + rlpCh = make(chan *numberRlp, threads*2) // we send raw rlp over this channel + hashesCh = make(chan *blockTxHashes, threads*2) // send hashes over hashesCh + abortCh = make(chan struct{}) + ) + // lookup runs in one instance + lookup := func() { + n, end := from, to + if reverse { + n, end = to-1, from-1 + } + defer close(rlpCh) + for n != end { + data := ReadCanonicalBodyRLP(db, n) + // Feed the block to the aggregator, or abort on interrupt + select { + case rlpCh <- &numberRlp{n, data}: + case <-abortCh: + return + } + if reverse { + n-- + } else { + n++ + } + } + } + // process runs in parallel + nThreadsAlive := int32(threads) + process := func() { + defer func() { + // Last processor closes the result channel + if atomic.AddInt32(&nThreadsAlive, -1) == 0 { + close(hashesCh) + } + }() + + var hasher = sha3.NewLegacyKeccak256() + for data := range rlpCh { + it, err := rlp.NewListIterator(data.rlp) + if err != nil { + log.Warn("tx iteration error", "error", err) + return + } + it.Next() + txs := it.Value() + txIt, err := rlp.NewListIterator(txs) + if err != nil { + log.Warn("tx iteration error", "error", err) + return + } + var hashes []common.Hash + for txIt.Next() { + if err := txIt.Err(); err != nil { + log.Warn("tx iteration error", "error", err) + return + } + var txHash common.Hash + hasher.Reset() + hasher.Write(txIt.Value()) + hasher.Sum(txHash[:0]) + hashes = append(hashes, txHash) + } + result := &blockTxHashes{ + hashes: hashes, + number: data.number, + } + // Feed the block to the aggregator, or abort on interrupt + select { + case hashesCh <- result: + case <-abortCh: + return + } + } + } + go lookup() // start the sequential db accessor + for i := 0; i < int(threads); i++ { + go process() + } + return hashesCh, abortCh +} + +// IndexTransactions creates txlookup indices of the specified block range. +// +// This function iterates canonical chain in reverse order, it has one main advantage: +// We can write tx index tail flag periodically even without the whole indexing +// procedure is finished. So that we can resume indexing procedure next time quickly. +func IndexTransactions(db ethdb.Database, from uint64, to uint64) { + // short circuit for invalid range + if from >= to { + return + } + var ( + hashesCh, abortCh = iterateTransactions(db, from, to, true) + batch = db.NewBatch() + start = time.Now() + logged = start.Add(-7 * time.Second) + // Since we iterate in reverse, we expect the first number to come + // in to be [to-1]. Therefore, setting lastNum to means that the + // prqueue gap-evaluation will work correctly + lastNum = to + queue = prque.New(nil) + // for stats reporting + blocks, txs = 0, 0 + ) + defer close(abortCh) + + for chanDelivery := range hashesCh { + // Push the delivery into the queue and process contiguous ranges. + // Since we iterate in reverse, so lower numbers have lower prio, and + // we can use the number directly as prio marker + queue.Push(chanDelivery, int64(chanDelivery.number)) + for !queue.Empty() { + // If the next available item is gapped, return + if _, priority := queue.Peek(); priority != int64(lastNum-1) { + break + } + // Next block available, pop it off and index it + delivery := queue.PopItem().(*blockTxHashes) + lastNum = delivery.number + WriteTxLookupEntries(batch, delivery.number, delivery.hashes) + blocks++ + txs += len(delivery.hashes) + // If enough data was accumulated in memory or we're at the last block, dump to disk + if batch.ValueSize() > ethdb.IdealBatchSize { + // Also write the tail there + WriteTxIndexTail(batch, lastNum) + if err := batch.Write(); err != nil { + log.Crit("Failed writing batch to db", "error", err) + return + } + batch.Reset() + } + // If we've spent too much time already, notify the user of what we're doing + if time.Since(logged) > 8*time.Second { + log.Info("Indexing transactions", "blocks", blocks, "txs", txs, "tail", lastNum, "total", to-from, "elapsed", common.PrettyDuration(time.Since(start))) + logged = time.Now() + } + } + } + if lastNum < to { + WriteTxIndexTail(batch, lastNum) + // No need to write the batch if we never entered the loop above... + if err := batch.Write(); err != nil { + log.Crit("Failed writing batch to db", "error", err) + return + } + } + log.Info("Indexed transactions", "blocks", blocks, "txs", txs, "tail", lastNum, "elapsed", common.PrettyDuration(time.Since(start))) +} + +// UnindexTransactions removes txlookup indices of the specified block range. +func UnindexTransactions(db ethdb.Database, from uint64, to uint64) { + // short circuit for invalid range + if from >= to { + return + } + // Write flag first and then unindex the transaction indices. Some indices + // will be left in the database if crash happens but it's fine. + WriteTxIndexTail(db, to) + // If only one block is unindexed, do it directly + //if from+1 == to { + // data := ReadCanonicalBodyRLP(db, uint64(from)) + // DeleteTxLookupEntries(db, ReadBlock(db, ReadCanonicalHash(db, from), from)) + // log.Info("Unindexed transactions", "blocks", 1, "tail", to) + // return + //} + // TODO @holiman, add this back (if we want it) + var ( + hashesCh, abortCh = iterateTransactions(db, from, to, false) + batch = db.NewBatch() + start = time.Now() + logged = start.Add(-7 * time.Second) + ) + defer close(abortCh) + // Otherwise spin up the concurrent iterator and unindexer + blocks, txs := 0, 0 + for delivery := range hashesCh { + DeleteTxLookupEntries(batch, delivery.hashes) + txs += len(delivery.hashes) + blocks++ + + // If enough data was accumulated in memory or we're at the last block, dump to disk + // A batch counts the size of deletion as '1', so we need to flush more + // often than that. + if blocks%1000 == 0 { + if err := batch.Write(); err != nil { + log.Crit("Failed writing batch to db", "error", err) + return + } + batch.Reset() + } + // If we've spent too much time already, notify the user of what we're doing + if time.Since(logged) > 8*time.Second { + log.Info("Unindexing transactions", "blocks", blocks, "txs", txs, "total", to-from, "elapsed", common.PrettyDuration(time.Since(start))) + logged = time.Now() + } + } + if err := batch.Write(); err != nil { + log.Crit("Failed writing batch to db", "error", err) + return + } + log.Info("Unindexed transactions", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start))) +} diff --git a/core/state/snapshot/difflayer_test.go b/core/state/snapshot/difflayer_test.go deleted file mode 100644 index 31636ee..0000000 --- a/core/state/snapshot/difflayer_test.go +++ /dev/null @@ -1,400 +0,0 @@ -// Copyright 2019 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package snapshot - -import ( - "bytes" - "math/rand" - "testing" - - "github.com/VictoriaMetrics/fastcache" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethdb/memorydb" -) - -func copyDestructs(destructs map[common.Hash]struct{}) map[common.Hash]struct{} { - copy := make(map[common.Hash]struct{}) - for hash := range destructs { - copy[hash] = struct{}{} - } - return copy -} - -func copyAccounts(accounts map[common.Hash][]byte) map[common.Hash][]byte { - copy := make(map[common.Hash][]byte) - for hash, blob := range accounts { - copy[hash] = blob - } - return copy -} - -func copyStorage(storage map[common.Hash]map[common.Hash][]byte) map[common.Hash]map[common.Hash][]byte { - copy := make(map[common.Hash]map[common.Hash][]byte) - for accHash, slots := range storage { - copy[accHash] = make(map[common.Hash][]byte) - for slotHash, blob := range slots { - copy[accHash][slotHash] = blob - } - } - return copy -} - -// TestMergeBasics tests some simple merges -func TestMergeBasics(t *testing.T) { - var ( - destructs = make(map[common.Hash]struct{}) - accounts = make(map[common.Hash][]byte) - storage = make(map[common.Hash]map[common.Hash][]byte) - ) - // Fill up a parent - for i := 0; i < 100; i++ { - h := randomHash() - data := randomAccount() - - accounts[h] = data - if rand.Intn(4) == 0 { - destructs[h] = struct{}{} - } - if rand.Intn(2) == 0 { - accStorage := make(map[common.Hash][]byte) - value := make([]byte, 32) - rand.Read(value) - accStorage[randomHash()] = value - storage[h] = accStorage - } - } - // Add some (identical) layers on top - parent := newDiffLayer(emptyLayer(), common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage)) - child := newDiffLayer(parent, common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage)) - child = newDiffLayer(child, common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage)) - child = newDiffLayer(child, common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage)) - child = newDiffLayer(child, common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage)) - // And flatten - merged := (child.flatten()).(*diffLayer) - - { // Check account lists - if have, want := len(merged.accountList), 0; have != want { - t.Errorf("accountList wrong: have %v, want %v", have, want) - } - if have, want := len(merged.AccountList()), len(accounts); have != want { - t.Errorf("AccountList() wrong: have %v, want %v", have, want) - } - if have, want := len(merged.accountList), len(accounts); have != want { - t.Errorf("accountList [2] wrong: have %v, want %v", have, want) - } - } - { // Check account drops - if have, want := len(merged.destructSet), len(destructs); have != want { - t.Errorf("accountDrop wrong: have %v, want %v", have, want) - } - } - { // Check storage lists - i := 0 - for aHash, sMap := range storage { - if have, want := len(merged.storageList), i; have != want { - t.Errorf("[1] storageList wrong: have %v, want %v", have, want) - } - list, _ := merged.StorageList(aHash) - if have, want := len(list), len(sMap); have != want { - t.Errorf("[2] StorageList() wrong: have %v, want %v", have, want) - } - if have, want := len(merged.storageList[aHash]), len(sMap); have != want { - t.Errorf("storageList wrong: have %v, want %v", have, want) - } - i++ - } - } -} - -// TestMergeDelete tests some deletion -func TestMergeDelete(t *testing.T) { - var ( - storage = make(map[common.Hash]map[common.Hash][]byte) - ) - // Fill up a parent - h1 := common.HexToHash("0x01") - h2 := common.HexToHash("0x02") - - flipDrops := func() map[common.Hash]struct{} { - return map[common.Hash]struct{}{ - h2: {}, - } - } - flipAccs := func() map[common.Hash][]byte { - return map[common.Hash][]byte{ - h1: randomAccount(), - } - } - flopDrops := func() map[common.Hash]struct{} { - return map[common.Hash]struct{}{ - h1: {}, - } - } - flopAccs := func() map[common.Hash][]byte { - return map[common.Hash][]byte{ - h2: randomAccount(), - } - } - // Add some flipAccs-flopping layers on top - parent := newDiffLayer(emptyLayer(), common.Hash{}, flipDrops(), flipAccs(), storage) - child := parent.Update(common.Hash{}, flopDrops(), flopAccs(), storage) - child = child.Update(common.Hash{}, flipDrops(), flipAccs(), storage) - child = child.Update(common.Hash{}, flopDrops(), flopAccs(), storage) - child = child.Update(common.Hash{}, flipDrops(), flipAccs(), storage) - child = child.Update(common.Hash{}, flopDrops(), flopAccs(), storage) - child = child.Update(common.Hash{}, flipDrops(), flipAccs(), storage) - - if data, _ := child.Account(h1); data == nil { - t.Errorf("last diff layer: expected %x account to be non-nil", h1) - } - if data, _ := child.Account(h2); data != nil { - t.Errorf("last diff layer: expected %x account to be nil", h2) - } - if _, ok := child.destructSet[h1]; ok { - t.Errorf("last diff layer: expected %x drop to be missing", h1) - } - if _, ok := child.destructSet[h2]; !ok { - t.Errorf("last diff layer: expected %x drop to be present", h1) - } - // And flatten - merged := (child.flatten()).(*diffLayer) - - if data, _ := merged.Account(h1); data == nil { - t.Errorf("merged layer: expected %x account to be non-nil", h1) - } - if data, _ := merged.Account(h2); data != nil { - t.Errorf("merged layer: expected %x account to be nil", h2) - } - if _, ok := merged.destructSet[h1]; !ok { // Note, drops stay alive until persisted to disk! - t.Errorf("merged diff layer: expected %x drop to be present", h1) - } - if _, ok := merged.destructSet[h2]; !ok { // Note, drops stay alive until persisted to disk! - t.Errorf("merged diff layer: expected %x drop to be present", h1) - } - // If we add more granular metering of memory, we can enable this again, - // but it's not implemented for now - //if have, want := merged.memory, child.memory; have != want { - // t.Errorf("mem wrong: have %d, want %d", have, want) - //} -} - -// This tests that if we create a new account, and set a slot, and then merge -// it, the lists will be correct. -func TestInsertAndMerge(t *testing.T) { - // Fill up a parent - var ( - acc = common.HexToHash("0x01") - slot = common.HexToHash("0x02") - parent *diffLayer - child *diffLayer - ) - { - var ( - destructs = make(map[common.Hash]struct{}) - accounts = make(map[common.Hash][]byte) - storage = make(map[common.Hash]map[common.Hash][]byte) - ) - parent = newDiffLayer(emptyLayer(), common.Hash{}, destructs, accounts, storage) - } - { - var ( - destructs = make(map[common.Hash]struct{}) - accounts = make(map[common.Hash][]byte) - storage = make(map[common.Hash]map[common.Hash][]byte) - ) - accounts[acc] = randomAccount() - storage[acc] = make(map[common.Hash][]byte) - storage[acc][slot] = []byte{0x01} - child = newDiffLayer(parent, common.Hash{}, destructs, accounts, storage) - } - // And flatten - merged := (child.flatten()).(*diffLayer) - { // Check that slot value is present - have, _ := merged.Storage(acc, slot) - if want := []byte{0x01}; !bytes.Equal(have, want) { - t.Errorf("merged slot value wrong: have %x, want %x", have, want) - } - } -} - -func emptyLayer() *diskLayer { - return &diskLayer{ - diskdb: memorydb.New(), - cache: fastcache.New(500 * 1024), - } -} - -// BenchmarkSearch checks how long it takes to find a non-existing key -// BenchmarkSearch-6 200000 10481 ns/op (1K per layer) -// BenchmarkSearch-6 200000 10760 ns/op (10K per layer) -// BenchmarkSearch-6 100000 17866 ns/op -// -// BenchmarkSearch-6 500000 3723 ns/op (10k per layer, only top-level RLock() -func BenchmarkSearch(b *testing.B) { - // First, we set up 128 diff layers, with 1K items each - fill := func(parent snapshot) *diffLayer { - var ( - destructs = make(map[common.Hash]struct{}) - accounts = make(map[common.Hash][]byte) - storage = make(map[common.Hash]map[common.Hash][]byte) - ) - for i := 0; i < 10000; i++ { - accounts[randomHash()] = randomAccount() - } - return newDiffLayer(parent, common.Hash{}, destructs, accounts, storage) - } - var layer snapshot - layer = emptyLayer() - for i := 0; i < 128; i++ { - layer = fill(layer) - } - key := crypto.Keccak256Hash([]byte{0x13, 0x38}) - b.ResetTimer() - for i := 0; i < b.N; i++ { - layer.AccountRLP(key) - } -} - -// BenchmarkSearchSlot checks how long it takes to find a non-existing key -// - Number of layers: 128 -// - Each layers contains the account, with a couple of storage slots -// BenchmarkSearchSlot-6 100000 14554 ns/op -// BenchmarkSearchSlot-6 100000 22254 ns/op (when checking parent root using mutex) -// BenchmarkSearchSlot-6 100000 14551 ns/op (when checking parent number using atomic) -// With bloom filter: -// BenchmarkSearchSlot-6 3467835 351 ns/op -func BenchmarkSearchSlot(b *testing.B) { - // First, we set up 128 diff layers, with 1K items each - accountKey := crypto.Keccak256Hash([]byte{0x13, 0x37}) - storageKey := crypto.Keccak256Hash([]byte{0x13, 0x37}) - accountRLP := randomAccount() - fill := func(parent snapshot) *diffLayer { - var ( - destructs = make(map[common.Hash]struct{}) - accounts = make(map[common.Hash][]byte) - storage = make(map[common.Hash]map[common.Hash][]byte) - ) - accounts[accountKey] = accountRLP - - accStorage := make(map[common.Hash][]byte) - for i := 0; i < 5; i++ { - value := make([]byte, 32) - rand.Read(value) - accStorage[randomHash()] = value - storage[accountKey] = accStorage - } - return newDiffLayer(parent, common.Hash{}, destructs, accounts, storage) - } - var layer snapshot - layer = emptyLayer() - for i := 0; i < 128; i++ { - layer = fill(layer) - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - layer.Storage(accountKey, storageKey) - } -} - -// With accountList and sorting -// BenchmarkFlatten-6 50 29890856 ns/op -// -// Without sorting and tracking accountlist -// BenchmarkFlatten-6 300 5511511 ns/op -func BenchmarkFlatten(b *testing.B) { - fill := func(parent snapshot) *diffLayer { - var ( - destructs = make(map[common.Hash]struct{}) - accounts = make(map[common.Hash][]byte) - storage = make(map[common.Hash]map[common.Hash][]byte) - ) - for i := 0; i < 100; i++ { - accountKey := randomHash() - accounts[accountKey] = randomAccount() - - accStorage := make(map[common.Hash][]byte) - for i := 0; i < 20; i++ { - value := make([]byte, 32) - rand.Read(value) - accStorage[randomHash()] = value - - } - storage[accountKey] = accStorage - } - return newDiffLayer(parent, common.Hash{}, destructs, accounts, storage) - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - b.StopTimer() - var layer snapshot - layer = emptyLayer() - for i := 1; i < 128; i++ { - layer = fill(layer) - } - b.StartTimer() - - for i := 1; i < 128; i++ { - dl, ok := layer.(*diffLayer) - if !ok { - break - } - layer = dl.flatten() - } - b.StopTimer() - } -} - -// This test writes ~324M of diff layers to disk, spread over -// - 128 individual layers, -// - each with 200 accounts -// - containing 200 slots -// -// BenchmarkJournal-6 1 1471373923 ns/ops -// BenchmarkJournal-6 1 1208083335 ns/op // bufio writer -func BenchmarkJournal(b *testing.B) { - fill := func(parent snapshot) *diffLayer { - var ( - destructs = make(map[common.Hash]struct{}) - accounts = make(map[common.Hash][]byte) - storage = make(map[common.Hash]map[common.Hash][]byte) - ) - for i := 0; i < 200; i++ { - accountKey := randomHash() - accounts[accountKey] = randomAccount() - - accStorage := make(map[common.Hash][]byte) - for i := 0; i < 200; i++ { - value := make([]byte, 32) - rand.Read(value) - accStorage[randomHash()] = value - - } - storage[accountKey] = accStorage - } - return newDiffLayer(parent, common.Hash{}, destructs, accounts, storage) - } - layer := snapshot(new(diskLayer)) - for i := 1; i < 128; i++ { - layer = fill(layer) - } - b.ResetTimer() - - for i := 0; i < b.N; i++ { - layer.Journal(new(bytes.Buffer)) - } -} diff --git a/core/state/snapshot/disklayer_test.go b/core/state/snapshot/disklayer_test.go deleted file mode 100644 index 5df5efc..0000000 --- a/core/state/snapshot/disklayer_test.go +++ /dev/null @@ -1,511 +0,0 @@ -// Copyright 2019 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package snapshot - -import ( - "bytes" - "io/ioutil" - "os" - "testing" - - "github.com/VictoriaMetrics/fastcache" - "github.com/ava-labs/coreth/core/rawdb" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/ethdb/leveldb" - "github.com/ethereum/go-ethereum/ethdb/memorydb" -) - -// reverse reverses the contents of a byte slice. It's used to update random accs -// with deterministic changes. -func reverse(blob []byte) []byte { - res := make([]byte, len(blob)) - for i, b := range blob { - res[len(blob)-1-i] = b - } - return res -} - -// Tests that merging something into a disk layer persists it into the database -// and invalidates any previously written and cached values. -func TestDiskMerge(t *testing.T) { - // Create some accounts in the disk layer - db := memorydb.New() - - var ( - accNoModNoCache = common.Hash{0x1} - accNoModCache = common.Hash{0x2} - accModNoCache = common.Hash{0x3} - accModCache = common.Hash{0x4} - accDelNoCache = common.Hash{0x5} - accDelCache = common.Hash{0x6} - conNoModNoCache = common.Hash{0x7} - conNoModNoCacheSlot = common.Hash{0x70} - conNoModCache = common.Hash{0x8} - conNoModCacheSlot = common.Hash{0x80} - conModNoCache = common.Hash{0x9} - conModNoCacheSlot = common.Hash{0x90} - conModCache = common.Hash{0xa} - conModCacheSlot = common.Hash{0xa0} - conDelNoCache = common.Hash{0xb} - conDelNoCacheSlot = common.Hash{0xb0} - conDelCache = common.Hash{0xc} - conDelCacheSlot = common.Hash{0xc0} - conNukeNoCache = common.Hash{0xd} - conNukeNoCacheSlot = common.Hash{0xd0} - conNukeCache = common.Hash{0xe} - conNukeCacheSlot = common.Hash{0xe0} - baseRoot = randomHash() - diffRoot = randomHash() - ) - - rawdb.WriteAccountSnapshot(db, accNoModNoCache, accNoModNoCache[:]) - rawdb.WriteAccountSnapshot(db, accNoModCache, accNoModCache[:]) - rawdb.WriteAccountSnapshot(db, accModNoCache, accModNoCache[:]) - rawdb.WriteAccountSnapshot(db, accModCache, accModCache[:]) - rawdb.WriteAccountSnapshot(db, accDelNoCache, accDelNoCache[:]) - rawdb.WriteAccountSnapshot(db, accDelCache, accDelCache[:]) - - rawdb.WriteAccountSnapshot(db, conNoModNoCache, conNoModNoCache[:]) - rawdb.WriteStorageSnapshot(db, conNoModNoCache, conNoModNoCacheSlot, conNoModNoCacheSlot[:]) - rawdb.WriteAccountSnapshot(db, conNoModCache, conNoModCache[:]) - rawdb.WriteStorageSnapshot(db, conNoModCache, conNoModCacheSlot, conNoModCacheSlot[:]) - rawdb.WriteAccountSnapshot(db, conModNoCache, conModNoCache[:]) - rawdb.WriteStorageSnapshot(db, conModNoCache, conModNoCacheSlot, conModNoCacheSlot[:]) - rawdb.WriteAccountSnapshot(db, conModCache, conModCache[:]) - rawdb.WriteStorageSnapshot(db, conModCache, conModCacheSlot, conModCacheSlot[:]) - rawdb.WriteAccountSnapshot(db, conDelNoCache, conDelNoCache[:]) - rawdb.WriteStorageSnapshot(db, conDelNoCache, conDelNoCacheSlot, conDelNoCacheSlot[:]) - rawdb.WriteAccountSnapshot(db, conDelCache, conDelCache[:]) - rawdb.WriteStorageSnapshot(db, conDelCache, conDelCacheSlot, conDelCacheSlot[:]) - - rawdb.WriteAccountSnapshot(db, conNukeNoCache, conNukeNoCache[:]) - rawdb.WriteStorageSnapshot(db, conNukeNoCache, conNukeNoCacheSlot, conNukeNoCacheSlot[:]) - rawdb.WriteAccountSnapshot(db, conNukeCache, conNukeCache[:]) - rawdb.WriteStorageSnapshot(db, conNukeCache, conNukeCacheSlot, conNukeCacheSlot[:]) - - rawdb.WriteSnapshotRoot(db, baseRoot) - - // Create a disk layer based on the above and cache in some data - snaps := &Tree{ - layers: map[common.Hash]snapshot{ - baseRoot: &diskLayer{ - diskdb: db, - cache: fastcache.New(500 * 1024), - root: baseRoot, - }, - }, - } - base := snaps.Snapshot(baseRoot) - base.AccountRLP(accNoModCache) - base.AccountRLP(accModCache) - base.AccountRLP(accDelCache) - base.Storage(conNoModCache, conNoModCacheSlot) - base.Storage(conModCache, conModCacheSlot) - base.Storage(conDelCache, conDelCacheSlot) - base.Storage(conNukeCache, conNukeCacheSlot) - - // Modify or delete some accounts, flatten everything onto disk - if err := snaps.Update(diffRoot, baseRoot, map[common.Hash]struct{}{ - accDelNoCache: {}, - accDelCache: {}, - conNukeNoCache: {}, - conNukeCache: {}, - }, map[common.Hash][]byte{ - accModNoCache: reverse(accModNoCache[:]), - accModCache: reverse(accModCache[:]), - }, map[common.Hash]map[common.Hash][]byte{ - conModNoCache: {conModNoCacheSlot: reverse(conModNoCacheSlot[:])}, - conModCache: {conModCacheSlot: reverse(conModCacheSlot[:])}, - conDelNoCache: {conDelNoCacheSlot: nil}, - conDelCache: {conDelCacheSlot: nil}, - }); err != nil { - t.Fatalf("failed to update snapshot tree: %v", err) - } - if err := snaps.Cap(diffRoot, 0); err != nil { - t.Fatalf("failed to flatten snapshot tree: %v", err) - } - // Retrieve all the data through the disk layer and validate it - base = snaps.Snapshot(diffRoot) - if _, ok := base.(*diskLayer); !ok { - t.Fatalf("update not flattend into the disk layer") - } - - // assertAccount ensures that an account matches the given blob. - assertAccount := func(account common.Hash, data []byte) { - t.Helper() - blob, err := base.AccountRLP(account) - if err != nil { - t.Errorf("account access (%x) failed: %v", account, err) - } else if !bytes.Equal(blob, data) { - t.Errorf("account access (%x) mismatch: have %x, want %x", account, blob, data) - } - } - assertAccount(accNoModNoCache, accNoModNoCache[:]) - assertAccount(accNoModCache, accNoModCache[:]) - assertAccount(accModNoCache, reverse(accModNoCache[:])) - assertAccount(accModCache, reverse(accModCache[:])) - assertAccount(accDelNoCache, nil) - assertAccount(accDelCache, nil) - - // assertStorage ensures that a storage slot matches the given blob. - assertStorage := func(account common.Hash, slot common.Hash, data []byte) { - t.Helper() - blob, err := base.Storage(account, slot) - if err != nil { - t.Errorf("storage access (%x:%x) failed: %v", account, slot, err) - } else if !bytes.Equal(blob, data) { - t.Errorf("storage access (%x:%x) mismatch: have %x, want %x", account, slot, blob, data) - } - } - assertStorage(conNoModNoCache, conNoModNoCacheSlot, conNoModNoCacheSlot[:]) - assertStorage(conNoModCache, conNoModCacheSlot, conNoModCacheSlot[:]) - assertStorage(conModNoCache, conModNoCacheSlot, reverse(conModNoCacheSlot[:])) - assertStorage(conModCache, conModCacheSlot, reverse(conModCacheSlot[:])) - assertStorage(conDelNoCache, conDelNoCacheSlot, nil) - assertStorage(conDelCache, conDelCacheSlot, nil) - assertStorage(conNukeNoCache, conNukeNoCacheSlot, nil) - assertStorage(conNukeCache, conNukeCacheSlot, nil) - - // Retrieve all the data directly from the database and validate it - - // assertDatabaseAccount ensures that an account from the database matches the given blob. - assertDatabaseAccount := func(account common.Hash, data []byte) { - t.Helper() - if blob := rawdb.ReadAccountSnapshot(db, account); !bytes.Equal(blob, data) { - t.Errorf("account database access (%x) mismatch: have %x, want %x", account, blob, data) - } - } - assertDatabaseAccount(accNoModNoCache, accNoModNoCache[:]) - assertDatabaseAccount(accNoModCache, accNoModCache[:]) - assertDatabaseAccount(accModNoCache, reverse(accModNoCache[:])) - assertDatabaseAccount(accModCache, reverse(accModCache[:])) - assertDatabaseAccount(accDelNoCache, nil) - assertDatabaseAccount(accDelCache, nil) - - // assertDatabaseStorage ensures that a storage slot from the database matches the given blob. - assertDatabaseStorage := func(account common.Hash, slot common.Hash, data []byte) { - t.Helper() - if blob := rawdb.ReadStorageSnapshot(db, account, slot); !bytes.Equal(blob, data) { - t.Errorf("storage database access (%x:%x) mismatch: have %x, want %x", account, slot, blob, data) - } - } - assertDatabaseStorage(conNoModNoCache, conNoModNoCacheSlot, conNoModNoCacheSlot[:]) - assertDatabaseStorage(conNoModCache, conNoModCacheSlot, conNoModCacheSlot[:]) - assertDatabaseStorage(conModNoCache, conModNoCacheSlot, reverse(conModNoCacheSlot[:])) - assertDatabaseStorage(conModCache, conModCacheSlot, reverse(conModCacheSlot[:])) - assertDatabaseStorage(conDelNoCache, conDelNoCacheSlot, nil) - assertDatabaseStorage(conDelCache, conDelCacheSlot, nil) - assertDatabaseStorage(conNukeNoCache, conNukeNoCacheSlot, nil) - assertDatabaseStorage(conNukeCache, conNukeCacheSlot, nil) -} - -// Tests that merging something into a disk layer persists it into the database -// and invalidates any previously written and cached values, discarding anything -// after the in-progress generation marker. -func TestDiskPartialMerge(t *testing.T) { - // Iterate the test a few times to ensure we pick various internal orderings - // for the data slots as well as the progress marker. - for i := 0; i < 1024; i++ { - // Create some accounts in the disk layer - db := memorydb.New() - - var ( - accNoModNoCache = randomHash() - accNoModCache = randomHash() - accModNoCache = randomHash() - accModCache = randomHash() - accDelNoCache = randomHash() - accDelCache = randomHash() - conNoModNoCache = randomHash() - conNoModNoCacheSlot = randomHash() - conNoModCache = randomHash() - conNoModCacheSlot = randomHash() - conModNoCache = randomHash() - conModNoCacheSlot = randomHash() - conModCache = randomHash() - conModCacheSlot = randomHash() - conDelNoCache = randomHash() - conDelNoCacheSlot = randomHash() - conDelCache = randomHash() - conDelCacheSlot = randomHash() - conNukeNoCache = randomHash() - conNukeNoCacheSlot = randomHash() - conNukeCache = randomHash() - conNukeCacheSlot = randomHash() - baseRoot = randomHash() - diffRoot = randomHash() - genMarker = append(randomHash().Bytes(), randomHash().Bytes()...) - ) - - // insertAccount injects an account into the database if it's after the - // generator marker, drops the op otherwise. This is needed to seed the - // database with a valid starting snapshot. - insertAccount := func(account common.Hash, data []byte) { - if bytes.Compare(account[:], genMarker) <= 0 { - rawdb.WriteAccountSnapshot(db, account, data[:]) - } - } - insertAccount(accNoModNoCache, accNoModNoCache[:]) - insertAccount(accNoModCache, accNoModCache[:]) - insertAccount(accModNoCache, accModNoCache[:]) - insertAccount(accModCache, accModCache[:]) - insertAccount(accDelNoCache, accDelNoCache[:]) - insertAccount(accDelCache, accDelCache[:]) - - // insertStorage injects a storage slot into the database if it's after - // the generator marker, drops the op otherwise. This is needed to seed - // the database with a valid starting snapshot. - insertStorage := func(account common.Hash, slot common.Hash, data []byte) { - if bytes.Compare(append(account[:], slot[:]...), genMarker) <= 0 { - rawdb.WriteStorageSnapshot(db, account, slot, data[:]) - } - } - insertAccount(conNoModNoCache, conNoModNoCache[:]) - insertStorage(conNoModNoCache, conNoModNoCacheSlot, conNoModNoCacheSlot[:]) - insertAccount(conNoModCache, conNoModCache[:]) - insertStorage(conNoModCache, conNoModCacheSlot, conNoModCacheSlot[:]) - insertAccount(conModNoCache, conModNoCache[:]) - insertStorage(conModNoCache, conModNoCacheSlot, conModNoCacheSlot[:]) - insertAccount(conModCache, conModCache[:]) - insertStorage(conModCache, conModCacheSlot, conModCacheSlot[:]) - insertAccount(conDelNoCache, conDelNoCache[:]) - insertStorage(conDelNoCache, conDelNoCacheSlot, conDelNoCacheSlot[:]) - insertAccount(conDelCache, conDelCache[:]) - insertStorage(conDelCache, conDelCacheSlot, conDelCacheSlot[:]) - - insertAccount(conNukeNoCache, conNukeNoCache[:]) - insertStorage(conNukeNoCache, conNukeNoCacheSlot, conNukeNoCacheSlot[:]) - insertAccount(conNukeCache, conNukeCache[:]) - insertStorage(conNukeCache, conNukeCacheSlot, conNukeCacheSlot[:]) - - rawdb.WriteSnapshotRoot(db, baseRoot) - - // Create a disk layer based on the above using a random progress marker - // and cache in some data. - snaps := &Tree{ - layers: map[common.Hash]snapshot{ - baseRoot: &diskLayer{ - diskdb: db, - cache: fastcache.New(500 * 1024), - root: baseRoot, - }, - }, - } - snaps.layers[baseRoot].(*diskLayer).genMarker = genMarker - base := snaps.Snapshot(baseRoot) - - // assertAccount ensures that an account matches the given blob if it's - // already covered by the disk snapshot, and errors out otherwise. - assertAccount := func(account common.Hash, data []byte) { - t.Helper() - blob, err := base.AccountRLP(account) - if bytes.Compare(account[:], genMarker) > 0 && err != ErrNotCoveredYet { - t.Fatalf("test %d: post-marker (%x) account access (%x) succeeded: %x", i, genMarker, account, blob) - } - if bytes.Compare(account[:], genMarker) <= 0 && !bytes.Equal(blob, data) { - t.Fatalf("test %d: pre-marker (%x) account access (%x) mismatch: have %x, want %x", i, genMarker, account, blob, data) - } - } - assertAccount(accNoModCache, accNoModCache[:]) - assertAccount(accModCache, accModCache[:]) - assertAccount(accDelCache, accDelCache[:]) - - // assertStorage ensures that a storage slot matches the given blob if - // it's already covered by the disk snapshot, and errors out otherwise. - assertStorage := func(account common.Hash, slot common.Hash, data []byte) { - t.Helper() - blob, err := base.Storage(account, slot) - if bytes.Compare(append(account[:], slot[:]...), genMarker) > 0 && err != ErrNotCoveredYet { - t.Fatalf("test %d: post-marker (%x) storage access (%x:%x) succeeded: %x", i, genMarker, account, slot, blob) - } - if bytes.Compare(append(account[:], slot[:]...), genMarker) <= 0 && !bytes.Equal(blob, data) { - t.Fatalf("test %d: pre-marker (%x) storage access (%x:%x) mismatch: have %x, want %x", i, genMarker, account, slot, blob, data) - } - } - assertStorage(conNoModCache, conNoModCacheSlot, conNoModCacheSlot[:]) - assertStorage(conModCache, conModCacheSlot, conModCacheSlot[:]) - assertStorage(conDelCache, conDelCacheSlot, conDelCacheSlot[:]) - assertStorage(conNukeCache, conNukeCacheSlot, conNukeCacheSlot[:]) - - // Modify or delete some accounts, flatten everything onto disk - if err := snaps.Update(diffRoot, baseRoot, map[common.Hash]struct{}{ - accDelNoCache: {}, - accDelCache: {}, - conNukeNoCache: {}, - conNukeCache: {}, - }, map[common.Hash][]byte{ - accModNoCache: reverse(accModNoCache[:]), - accModCache: reverse(accModCache[:]), - }, map[common.Hash]map[common.Hash][]byte{ - conModNoCache: {conModNoCacheSlot: reverse(conModNoCacheSlot[:])}, - conModCache: {conModCacheSlot: reverse(conModCacheSlot[:])}, - conDelNoCache: {conDelNoCacheSlot: nil}, - conDelCache: {conDelCacheSlot: nil}, - }); err != nil { - t.Fatalf("test %d: failed to update snapshot tree: %v", i, err) - } - if err := snaps.Cap(diffRoot, 0); err != nil { - t.Fatalf("test %d: failed to flatten snapshot tree: %v", i, err) - } - // Retrieve all the data through the disk layer and validate it - base = snaps.Snapshot(diffRoot) - if _, ok := base.(*diskLayer); !ok { - t.Fatalf("test %d: update not flattend into the disk layer", i) - } - assertAccount(accNoModNoCache, accNoModNoCache[:]) - assertAccount(accNoModCache, accNoModCache[:]) - assertAccount(accModNoCache, reverse(accModNoCache[:])) - assertAccount(accModCache, reverse(accModCache[:])) - assertAccount(accDelNoCache, nil) - assertAccount(accDelCache, nil) - - assertStorage(conNoModNoCache, conNoModNoCacheSlot, conNoModNoCacheSlot[:]) - assertStorage(conNoModCache, conNoModCacheSlot, conNoModCacheSlot[:]) - assertStorage(conModNoCache, conModNoCacheSlot, reverse(conModNoCacheSlot[:])) - assertStorage(conModCache, conModCacheSlot, reverse(conModCacheSlot[:])) - assertStorage(conDelNoCache, conDelNoCacheSlot, nil) - assertStorage(conDelCache, conDelCacheSlot, nil) - assertStorage(conNukeNoCache, conNukeNoCacheSlot, nil) - assertStorage(conNukeCache, conNukeCacheSlot, nil) - - // Retrieve all the data directly from the database and validate it - - // assertDatabaseAccount ensures that an account inside the database matches - // the given blob if it's already covered by the disk snapshot, and does not - // exist otherwise. - assertDatabaseAccount := func(account common.Hash, data []byte) { - t.Helper() - blob := rawdb.ReadAccountSnapshot(db, account) - if bytes.Compare(account[:], genMarker) > 0 && blob != nil { - t.Fatalf("test %d: post-marker (%x) account database access (%x) succeeded: %x", i, genMarker, account, blob) - } - if bytes.Compare(account[:], genMarker) <= 0 && !bytes.Equal(blob, data) { - t.Fatalf("test %d: pre-marker (%x) account database access (%x) mismatch: have %x, want %x", i, genMarker, account, blob, data) - } - } - assertDatabaseAccount(accNoModNoCache, accNoModNoCache[:]) - assertDatabaseAccount(accNoModCache, accNoModCache[:]) - assertDatabaseAccount(accModNoCache, reverse(accModNoCache[:])) - assertDatabaseAccount(accModCache, reverse(accModCache[:])) - assertDatabaseAccount(accDelNoCache, nil) - assertDatabaseAccount(accDelCache, nil) - - // assertDatabaseStorage ensures that a storage slot inside the database - // matches the given blob if it's already covered by the disk snapshot, - // and does not exist otherwise. - assertDatabaseStorage := func(account common.Hash, slot common.Hash, data []byte) { - t.Helper() - blob := rawdb.ReadStorageSnapshot(db, account, slot) - if bytes.Compare(append(account[:], slot[:]...), genMarker) > 0 && blob != nil { - t.Fatalf("test %d: post-marker (%x) storage database access (%x:%x) succeeded: %x", i, genMarker, account, slot, blob) - } - if bytes.Compare(append(account[:], slot[:]...), genMarker) <= 0 && !bytes.Equal(blob, data) { - t.Fatalf("test %d: pre-marker (%x) storage database access (%x:%x) mismatch: have %x, want %x", i, genMarker, account, slot, blob, data) - } - } - assertDatabaseStorage(conNoModNoCache, conNoModNoCacheSlot, conNoModNoCacheSlot[:]) - assertDatabaseStorage(conNoModCache, conNoModCacheSlot, conNoModCacheSlot[:]) - assertDatabaseStorage(conModNoCache, conModNoCacheSlot, reverse(conModNoCacheSlot[:])) - assertDatabaseStorage(conModCache, conModCacheSlot, reverse(conModCacheSlot[:])) - assertDatabaseStorage(conDelNoCache, conDelNoCacheSlot, nil) - assertDatabaseStorage(conDelCache, conDelCacheSlot, nil) - assertDatabaseStorage(conNukeNoCache, conNukeNoCacheSlot, nil) - assertDatabaseStorage(conNukeCache, conNukeCacheSlot, nil) - } -} - -// Tests that merging something into a disk layer persists it into the database -// and invalidates any previously written and cached values, discarding anything -// after the in-progress generation marker. -// -// This test case is a tiny specialized case of TestDiskPartialMerge, which tests -// some very specific cornercases that random tests won't ever trigger. -func TestDiskMidAccountPartialMerge(t *testing.T) { - // TODO(@karalabe) ? -} - -// TestDiskSeek tests that seek-operations work on the disk layer -func TestDiskSeek(t *testing.T) { - // Create some accounts in the disk layer - var db ethdb.Database - - if dir, err := ioutil.TempDir("", "disklayer-test"); err != nil { - t.Fatal(err) - } else { - defer os.RemoveAll(dir) - diskdb, err := leveldb.New(dir, 256, 0, "") - if err != nil { - t.Fatal(err) - } - db = rawdb.NewDatabase(diskdb) - } - // Fill even keys [0,2,4...] - for i := 0; i < 0xff; i += 2 { - acc := common.Hash{byte(i)} - rawdb.WriteAccountSnapshot(db, acc, acc[:]) - } - // Add an 'higher' key, with incorrect (higher) prefix - highKey := []byte{rawdb.SnapshotAccountPrefix[0] + 1} - db.Put(highKey, []byte{0xff, 0xff}) - - baseRoot := randomHash() - rawdb.WriteSnapshotRoot(db, baseRoot) - - snaps := &Tree{ - layers: map[common.Hash]snapshot{ - baseRoot: &diskLayer{ - diskdb: db, - cache: fastcache.New(500 * 1024), - root: baseRoot, - }, - }, - } - // Test some different seek positions - type testcase struct { - pos byte - expkey byte - } - var cases = []testcase{ - {0xff, 0x55}, // this should exit immediately without checking key - {0x01, 0x02}, - {0xfe, 0xfe}, - {0xfd, 0xfe}, - {0x00, 0x00}, - } - for i, tc := range cases { - it, err := snaps.AccountIterator(baseRoot, common.Hash{tc.pos}) - if err != nil { - t.Fatalf("case %d, error: %v", i, err) - } - count := 0 - for it.Next() { - k, v, err := it.Hash()[0], it.Account()[0], it.Error() - if err != nil { - t.Fatalf("test %d, item %d, error: %v", i, count, err) - } - // First item in iterator should have the expected key - if count == 0 && k != tc.expkey { - t.Fatalf("test %d, item %d, got %v exp %v", i, count, k, tc.expkey) - } - count++ - if v != k { - t.Fatalf("test %d, item %d, value wrong, got %v exp %v", i, count, v, k) - } - } - } -} diff --git a/core/state/snapshot/iterator_test.go b/core/state/snapshot/iterator_test.go deleted file mode 100644 index ef4859c..0000000 --- a/core/state/snapshot/iterator_test.go +++ /dev/null @@ -1,1046 +0,0 @@ -// Copyright 2019 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package snapshot - -import ( - "bytes" - "encoding/binary" - "fmt" - "math/rand" - "testing" - - "github.com/VictoriaMetrics/fastcache" - "github.com/ava-labs/coreth/core/rawdb" - "github.com/ethereum/go-ethereum/common" -) - -// TestAccountIteratorBasics tests some simple single-layer(diff and disk) iteration -func TestAccountIteratorBasics(t *testing.T) { - var ( - destructs = make(map[common.Hash]struct{}) - accounts = make(map[common.Hash][]byte) - storage = make(map[common.Hash]map[common.Hash][]byte) - ) - // Fill up a parent - for i := 0; i < 100; i++ { - h := randomHash() - data := randomAccount() - - accounts[h] = data - if rand.Intn(4) == 0 { - destructs[h] = struct{}{} - } - if rand.Intn(2) == 0 { - accStorage := make(map[common.Hash][]byte) - value := make([]byte, 32) - rand.Read(value) - accStorage[randomHash()] = value - storage[h] = accStorage - } - } - // Add some (identical) layers on top - diffLayer := newDiffLayer(emptyLayer(), common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage)) - it := diffLayer.AccountIterator(common.Hash{}) - verifyIterator(t, 100, it, verifyNothing) // Nil is allowed for single layer iterator - - diskLayer := diffToDisk(diffLayer) - it = diskLayer.AccountIterator(common.Hash{}) - verifyIterator(t, 100, it, verifyNothing) // Nil is allowed for single layer iterator -} - -// TestStorageIteratorBasics tests some simple single-layer(diff and disk) iteration for storage -func TestStorageIteratorBasics(t *testing.T) { - var ( - nilStorage = make(map[common.Hash]int) - accounts = make(map[common.Hash][]byte) - storage = make(map[common.Hash]map[common.Hash][]byte) - ) - // Fill some random data - for i := 0; i < 10; i++ { - h := randomHash() - accounts[h] = randomAccount() - - accStorage := make(map[common.Hash][]byte) - value := make([]byte, 32) - - var nilstorage int - for i := 0; i < 100; i++ { - rand.Read(value) - if rand.Intn(2) == 0 { - accStorage[randomHash()] = common.CopyBytes(value) - } else { - accStorage[randomHash()] = nil // delete slot - nilstorage += 1 - } - } - storage[h] = accStorage - nilStorage[h] = nilstorage - } - // Add some (identical) layers on top - diffLayer := newDiffLayer(emptyLayer(), common.Hash{}, nil, copyAccounts(accounts), copyStorage(storage)) - for account := range accounts { - it, _ := diffLayer.StorageIterator(account, common.Hash{}) - verifyIterator(t, 100, it, verifyNothing) // Nil is allowed for single layer iterator - } - - diskLayer := diffToDisk(diffLayer) - for account := range accounts { - it, _ := diskLayer.StorageIterator(account, common.Hash{}) - verifyIterator(t, 100-nilStorage[account], it, verifyNothing) // Nil is allowed for single layer iterator - } -} - -type testIterator struct { - values []byte -} - -func newTestIterator(values ...byte) *testIterator { - return &testIterator{values} -} - -func (ti *testIterator) Seek(common.Hash) { - panic("implement me") -} - -func (ti *testIterator) Next() bool { - ti.values = ti.values[1:] - return len(ti.values) > 0 -} - -func (ti *testIterator) Error() error { - return nil -} - -func (ti *testIterator) Hash() common.Hash { - return common.BytesToHash([]byte{ti.values[0]}) -} - -func (ti *testIterator) Account() []byte { - return nil -} - -func (ti *testIterator) Slot() []byte { - return nil -} - -func (ti *testIterator) Release() {} - -func TestFastIteratorBasics(t *testing.T) { - type testCase struct { - lists [][]byte - expKeys []byte - } - for i, tc := range []testCase{ - {lists: [][]byte{{0, 1, 8}, {1, 2, 8}, {2, 9}, {4}, - {7, 14, 15}, {9, 13, 15, 16}}, - expKeys: []byte{0, 1, 2, 4, 7, 8, 9, 13, 14, 15, 16}}, - {lists: [][]byte{{0, 8}, {1, 2, 8}, {7, 14, 15}, {8, 9}, - {9, 10}, {10, 13, 15, 16}}, - expKeys: []byte{0, 1, 2, 7, 8, 9, 10, 13, 14, 15, 16}}, - } { - var iterators []*weightedIterator - for i, data := range tc.lists { - it := newTestIterator(data...) - iterators = append(iterators, &weightedIterator{it, i}) - } - fi := &fastIterator{ - iterators: iterators, - initiated: false, - } - count := 0 - for fi.Next() { - if got, exp := fi.Hash()[31], tc.expKeys[count]; exp != got { - t.Errorf("tc %d, [%d]: got %d exp %d", i, count, got, exp) - } - count++ - } - } -} - -type verifyContent int - -const ( - verifyNothing verifyContent = iota - verifyAccount - verifyStorage -) - -func verifyIterator(t *testing.T, expCount int, it Iterator, verify verifyContent) { - t.Helper() - - var ( - count = 0 - last = common.Hash{} - ) - for it.Next() { - hash := it.Hash() - if bytes.Compare(last[:], hash[:]) >= 0 { - t.Errorf("wrong order: %x >= %x", last, hash) - } - count++ - if verify == verifyAccount && len(it.(AccountIterator).Account()) == 0 { - t.Errorf("iterator returned nil-value for hash %x", hash) - } else if verify == verifyStorage && len(it.(StorageIterator).Slot()) == 0 { - t.Errorf("iterator returned nil-value for hash %x", hash) - } - last = hash - } - if count != expCount { - t.Errorf("iterator count mismatch: have %d, want %d", count, expCount) - } - if err := it.Error(); err != nil { - t.Errorf("iterator failed: %v", err) - } -} - -// TestAccountIteratorTraversal tests some simple multi-layer iteration. -func TestAccountIteratorTraversal(t *testing.T) { - // Create an empty base layer and a snapshot tree out of it - base := &diskLayer{ - diskdb: rawdb.NewMemoryDatabase(), - root: common.HexToHash("0x01"), - cache: fastcache.New(1024 * 500), - } - snaps := &Tree{ - layers: map[common.Hash]snapshot{ - base.root: base, - }, - } - // Stack three diff layers on top with various overlaps - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, - randomAccountSet("0xaa", "0xee", "0xff", "0xf0"), nil) - - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, - randomAccountSet("0xbb", "0xdd", "0xf0"), nil) - - snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, - randomAccountSet("0xcc", "0xf0", "0xff"), nil) - - // Verify the single and multi-layer iterators - head := snaps.Snapshot(common.HexToHash("0x04")) - - verifyIterator(t, 3, head.(snapshot).AccountIterator(common.Hash{}), verifyNothing) - verifyIterator(t, 7, head.(*diffLayer).newBinaryAccountIterator(), verifyAccount) - - it, _ := snaps.AccountIterator(common.HexToHash("0x04"), common.Hash{}) - verifyIterator(t, 7, it, verifyAccount) - it.Release() - - // Test after persist some bottom-most layers into the disk, - // the functionalities still work. - limit := aggregatorMemoryLimit - defer func() { - aggregatorMemoryLimit = limit - }() - aggregatorMemoryLimit = 0 // Force pushing the bottom-most layer into disk - snaps.Cap(common.HexToHash("0x04"), 2) - verifyIterator(t, 7, head.(*diffLayer).newBinaryAccountIterator(), verifyAccount) - - it, _ = snaps.AccountIterator(common.HexToHash("0x04"), common.Hash{}) - verifyIterator(t, 7, it, verifyAccount) - it.Release() -} - -func TestStorageIteratorTraversal(t *testing.T) { - // Create an empty base layer and a snapshot tree out of it - base := &diskLayer{ - diskdb: rawdb.NewMemoryDatabase(), - root: common.HexToHash("0x01"), - cache: fastcache.New(1024 * 500), - } - snaps := &Tree{ - layers: map[common.Hash]snapshot{ - base.root: base, - }, - } - // Stack three diff layers on top with various overlaps - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, - randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x01", "0x02", "0x03"}}, nil)) - - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, - randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x04", "0x05", "0x06"}}, nil)) - - snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, - randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x01", "0x02", "0x03"}}, nil)) - - // Verify the single and multi-layer iterators - head := snaps.Snapshot(common.HexToHash("0x04")) - - diffIter, _ := head.(snapshot).StorageIterator(common.HexToHash("0xaa"), common.Hash{}) - verifyIterator(t, 3, diffIter, verifyNothing) - verifyIterator(t, 6, head.(*diffLayer).newBinaryStorageIterator(common.HexToHash("0xaa")), verifyStorage) - - it, _ := snaps.StorageIterator(common.HexToHash("0x04"), common.HexToHash("0xaa"), common.Hash{}) - verifyIterator(t, 6, it, verifyStorage) - it.Release() - - // Test after persist some bottom-most layers into the disk, - // the functionalities still work. - limit := aggregatorMemoryLimit - defer func() { - aggregatorMemoryLimit = limit - }() - aggregatorMemoryLimit = 0 // Force pushing the bottom-most layer into disk - snaps.Cap(common.HexToHash("0x04"), 2) - verifyIterator(t, 6, head.(*diffLayer).newBinaryStorageIterator(common.HexToHash("0xaa")), verifyStorage) - - it, _ = snaps.StorageIterator(common.HexToHash("0x04"), common.HexToHash("0xaa"), common.Hash{}) - verifyIterator(t, 6, it, verifyStorage) - it.Release() -} - -// TestAccountIteratorTraversalValues tests some multi-layer iteration, where we -// also expect the correct values to show up. -func TestAccountIteratorTraversalValues(t *testing.T) { - // Create an empty base layer and a snapshot tree out of it - base := &diskLayer{ - diskdb: rawdb.NewMemoryDatabase(), - root: common.HexToHash("0x01"), - cache: fastcache.New(1024 * 500), - } - snaps := &Tree{ - layers: map[common.Hash]snapshot{ - base.root: base, - }, - } - // Create a batch of account sets to seed subsequent layers with - var ( - a = make(map[common.Hash][]byte) - b = make(map[common.Hash][]byte) - c = make(map[common.Hash][]byte) - d = make(map[common.Hash][]byte) - e = make(map[common.Hash][]byte) - f = make(map[common.Hash][]byte) - g = make(map[common.Hash][]byte) - h = make(map[common.Hash][]byte) - ) - for i := byte(2); i < 0xff; i++ { - a[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 0, i)) - if i > 20 && i%2 == 0 { - b[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 1, i)) - } - if i%4 == 0 { - c[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 2, i)) - } - if i%7 == 0 { - d[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 3, i)) - } - if i%8 == 0 { - e[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 4, i)) - } - if i > 50 || i < 85 { - f[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 5, i)) - } - if i%64 == 0 { - g[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 6, i)) - } - if i%128 == 0 { - h[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 7, i)) - } - } - // Assemble a stack of snapshots from the account layers - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, a, nil) - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, b, nil) - snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, c, nil) - snaps.Update(common.HexToHash("0x05"), common.HexToHash("0x04"), nil, d, nil) - snaps.Update(common.HexToHash("0x06"), common.HexToHash("0x05"), nil, e, nil) - snaps.Update(common.HexToHash("0x07"), common.HexToHash("0x06"), nil, f, nil) - snaps.Update(common.HexToHash("0x08"), common.HexToHash("0x07"), nil, g, nil) - snaps.Update(common.HexToHash("0x09"), common.HexToHash("0x08"), nil, h, nil) - - it, _ := snaps.AccountIterator(common.HexToHash("0x09"), common.Hash{}) - head := snaps.Snapshot(common.HexToHash("0x09")) - for it.Next() { - hash := it.Hash() - want, err := head.AccountRLP(hash) - if err != nil { - t.Fatalf("failed to retrieve expected account: %v", err) - } - if have := it.Account(); !bytes.Equal(want, have) { - t.Fatalf("hash %x: account mismatch: have %x, want %x", hash, have, want) - } - } - it.Release() - - // Test after persist some bottom-most layers into the disk, - // the functionalities still work. - limit := aggregatorMemoryLimit - defer func() { - aggregatorMemoryLimit = limit - }() - aggregatorMemoryLimit = 0 // Force pushing the bottom-most layer into disk - snaps.Cap(common.HexToHash("0x09"), 2) - - it, _ = snaps.AccountIterator(common.HexToHash("0x09"), common.Hash{}) - for it.Next() { - hash := it.Hash() - want, err := head.AccountRLP(hash) - if err != nil { - t.Fatalf("failed to retrieve expected account: %v", err) - } - if have := it.Account(); !bytes.Equal(want, have) { - t.Fatalf("hash %x: account mismatch: have %x, want %x", hash, have, want) - } - } - it.Release() -} - -func TestStorageIteratorTraversalValues(t *testing.T) { - // Create an empty base layer and a snapshot tree out of it - base := &diskLayer{ - diskdb: rawdb.NewMemoryDatabase(), - root: common.HexToHash("0x01"), - cache: fastcache.New(1024 * 500), - } - snaps := &Tree{ - layers: map[common.Hash]snapshot{ - base.root: base, - }, - } - wrapStorage := func(storage map[common.Hash][]byte) map[common.Hash]map[common.Hash][]byte { - return map[common.Hash]map[common.Hash][]byte{ - common.HexToHash("0xaa"): storage, - } - } - // Create a batch of storage sets to seed subsequent layers with - var ( - a = make(map[common.Hash][]byte) - b = make(map[common.Hash][]byte) - c = make(map[common.Hash][]byte) - d = make(map[common.Hash][]byte) - e = make(map[common.Hash][]byte) - f = make(map[common.Hash][]byte) - g = make(map[common.Hash][]byte) - h = make(map[common.Hash][]byte) - ) - for i := byte(2); i < 0xff; i++ { - a[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 0, i)) - if i > 20 && i%2 == 0 { - b[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 1, i)) - } - if i%4 == 0 { - c[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 2, i)) - } - if i%7 == 0 { - d[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 3, i)) - } - if i%8 == 0 { - e[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 4, i)) - } - if i > 50 || i < 85 { - f[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 5, i)) - } - if i%64 == 0 { - g[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 6, i)) - } - if i%128 == 0 { - h[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 7, i)) - } - } - // Assemble a stack of snapshots from the account layers - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, randomAccountSet("0xaa"), wrapStorage(a)) - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, randomAccountSet("0xaa"), wrapStorage(b)) - snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, randomAccountSet("0xaa"), wrapStorage(c)) - snaps.Update(common.HexToHash("0x05"), common.HexToHash("0x04"), nil, randomAccountSet("0xaa"), wrapStorage(d)) - snaps.Update(common.HexToHash("0x06"), common.HexToHash("0x05"), nil, randomAccountSet("0xaa"), wrapStorage(e)) - snaps.Update(common.HexToHash("0x07"), common.HexToHash("0x06"), nil, randomAccountSet("0xaa"), wrapStorage(e)) - snaps.Update(common.HexToHash("0x08"), common.HexToHash("0x07"), nil, randomAccountSet("0xaa"), wrapStorage(g)) - snaps.Update(common.HexToHash("0x09"), common.HexToHash("0x08"), nil, randomAccountSet("0xaa"), wrapStorage(h)) - - it, _ := snaps.StorageIterator(common.HexToHash("0x09"), common.HexToHash("0xaa"), common.Hash{}) - head := snaps.Snapshot(common.HexToHash("0x09")) - for it.Next() { - hash := it.Hash() - want, err := head.Storage(common.HexToHash("0xaa"), hash) - if err != nil { - t.Fatalf("failed to retrieve expected storage slot: %v", err) - } - if have := it.Slot(); !bytes.Equal(want, have) { - t.Fatalf("hash %x: slot mismatch: have %x, want %x", hash, have, want) - } - } - it.Release() - - // Test after persist some bottom-most layers into the disk, - // the functionalities still work. - limit := aggregatorMemoryLimit - defer func() { - aggregatorMemoryLimit = limit - }() - aggregatorMemoryLimit = 0 // Force pushing the bottom-most layer into disk - snaps.Cap(common.HexToHash("0x09"), 2) - - it, _ = snaps.StorageIterator(common.HexToHash("0x09"), common.HexToHash("0xaa"), common.Hash{}) - for it.Next() { - hash := it.Hash() - want, err := head.Storage(common.HexToHash("0xaa"), hash) - if err != nil { - t.Fatalf("failed to retrieve expected slot: %v", err) - } - if have := it.Slot(); !bytes.Equal(want, have) { - t.Fatalf("hash %x: slot mismatch: have %x, want %x", hash, have, want) - } - } - it.Release() -} - -// This testcase is notorious, all layers contain the exact same 200 accounts. -func TestAccountIteratorLargeTraversal(t *testing.T) { - // Create a custom account factory to recreate the same addresses - makeAccounts := func(num int) map[common.Hash][]byte { - accounts := make(map[common.Hash][]byte) - for i := 0; i < num; i++ { - h := common.Hash{} - binary.BigEndian.PutUint64(h[:], uint64(i+1)) - accounts[h] = randomAccount() - } - return accounts - } - // Build up a large stack of snapshots - base := &diskLayer{ - diskdb: rawdb.NewMemoryDatabase(), - root: common.HexToHash("0x01"), - cache: fastcache.New(1024 * 500), - } - snaps := &Tree{ - layers: map[common.Hash]snapshot{ - base.root: base, - }, - } - for i := 1; i < 128; i++ { - snaps.Update(common.HexToHash(fmt.Sprintf("0x%02x", i+1)), common.HexToHash(fmt.Sprintf("0x%02x", i)), nil, makeAccounts(200), nil) - } - // Iterate the entire stack and ensure everything is hit only once - head := snaps.Snapshot(common.HexToHash("0x80")) - verifyIterator(t, 200, head.(snapshot).AccountIterator(common.Hash{}), verifyNothing) - verifyIterator(t, 200, head.(*diffLayer).newBinaryAccountIterator(), verifyAccount) - - it, _ := snaps.AccountIterator(common.HexToHash("0x80"), common.Hash{}) - verifyIterator(t, 200, it, verifyAccount) - it.Release() - - // Test after persist some bottom-most layers into the disk, - // the functionalities still work. - limit := aggregatorMemoryLimit - defer func() { - aggregatorMemoryLimit = limit - }() - aggregatorMemoryLimit = 0 // Force pushing the bottom-most layer into disk - snaps.Cap(common.HexToHash("0x80"), 2) - - verifyIterator(t, 200, head.(*diffLayer).newBinaryAccountIterator(), verifyAccount) - - it, _ = snaps.AccountIterator(common.HexToHash("0x80"), common.Hash{}) - verifyIterator(t, 200, it, verifyAccount) - it.Release() -} - -// TestAccountIteratorFlattening tests what happens when we -// - have a live iterator on child C (parent C1 -> C2 .. CN) -// - flattens C2 all the way into CN -// - continues iterating -func TestAccountIteratorFlattening(t *testing.T) { - // Create an empty base layer and a snapshot tree out of it - base := &diskLayer{ - diskdb: rawdb.NewMemoryDatabase(), - root: common.HexToHash("0x01"), - cache: fastcache.New(1024 * 500), - } - snaps := &Tree{ - layers: map[common.Hash]snapshot{ - base.root: base, - }, - } - // Create a stack of diffs on top - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, - randomAccountSet("0xaa", "0xee", "0xff", "0xf0"), nil) - - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, - randomAccountSet("0xbb", "0xdd", "0xf0"), nil) - - snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, - randomAccountSet("0xcc", "0xf0", "0xff"), nil) - - // Create an iterator and flatten the data from underneath it - it, _ := snaps.AccountIterator(common.HexToHash("0x04"), common.Hash{}) - defer it.Release() - - if err := snaps.Cap(common.HexToHash("0x04"), 1); err != nil { - t.Fatalf("failed to flatten snapshot stack: %v", err) - } - //verifyIterator(t, 7, it) -} - -func TestAccountIteratorSeek(t *testing.T) { - // Create a snapshot stack with some initial data - base := &diskLayer{ - diskdb: rawdb.NewMemoryDatabase(), - root: common.HexToHash("0x01"), - cache: fastcache.New(1024 * 500), - } - snaps := &Tree{ - layers: map[common.Hash]snapshot{ - base.root: base, - }, - } - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, - randomAccountSet("0xaa", "0xee", "0xff", "0xf0"), nil) - - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, - randomAccountSet("0xbb", "0xdd", "0xf0"), nil) - - snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, - randomAccountSet("0xcc", "0xf0", "0xff"), nil) - - // Account set is now - // 02: aa, ee, f0, ff - // 03: aa, bb, dd, ee, f0 (, f0), ff - // 04: aa, bb, cc, dd, ee, f0 (, f0), ff (, ff) - // Construct various iterators and ensure their traversal is correct - it, _ := snaps.AccountIterator(common.HexToHash("0x02"), common.HexToHash("0xdd")) - defer it.Release() - verifyIterator(t, 3, it, verifyAccount) // expected: ee, f0, ff - - it, _ = snaps.AccountIterator(common.HexToHash("0x02"), common.HexToHash("0xaa")) - defer it.Release() - verifyIterator(t, 4, it, verifyAccount) // expected: aa, ee, f0, ff - - it, _ = snaps.AccountIterator(common.HexToHash("0x02"), common.HexToHash("0xff")) - defer it.Release() - verifyIterator(t, 1, it, verifyAccount) // expected: ff - - it, _ = snaps.AccountIterator(common.HexToHash("0x02"), common.HexToHash("0xff1")) - defer it.Release() - verifyIterator(t, 0, it, verifyAccount) // expected: nothing - - it, _ = snaps.AccountIterator(common.HexToHash("0x04"), common.HexToHash("0xbb")) - defer it.Release() - verifyIterator(t, 6, it, verifyAccount) // expected: bb, cc, dd, ee, f0, ff - - it, _ = snaps.AccountIterator(common.HexToHash("0x04"), common.HexToHash("0xef")) - defer it.Release() - verifyIterator(t, 2, it, verifyAccount) // expected: f0, ff - - it, _ = snaps.AccountIterator(common.HexToHash("0x04"), common.HexToHash("0xf0")) - defer it.Release() - verifyIterator(t, 2, it, verifyAccount) // expected: f0, ff - - it, _ = snaps.AccountIterator(common.HexToHash("0x04"), common.HexToHash("0xff")) - defer it.Release() - verifyIterator(t, 1, it, verifyAccount) // expected: ff - - it, _ = snaps.AccountIterator(common.HexToHash("0x04"), common.HexToHash("0xff1")) - defer it.Release() - verifyIterator(t, 0, it, verifyAccount) // expected: nothing -} - -func TestStorageIteratorSeek(t *testing.T) { - // Create a snapshot stack with some initial data - base := &diskLayer{ - diskdb: rawdb.NewMemoryDatabase(), - root: common.HexToHash("0x01"), - cache: fastcache.New(1024 * 500), - } - snaps := &Tree{ - layers: map[common.Hash]snapshot{ - base.root: base, - }, - } - // Stack three diff layers on top with various overlaps - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, - randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x01", "0x03", "0x05"}}, nil)) - - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, - randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x02", "0x05", "0x06"}}, nil)) - - snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, - randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x01", "0x05", "0x08"}}, nil)) - - // Account set is now - // 02: 01, 03, 05 - // 03: 01, 02, 03, 05 (, 05), 06 - // 04: 01(, 01), 02, 03, 05(, 05, 05), 06, 08 - // Construct various iterators and ensure their traversal is correct - it, _ := snaps.StorageIterator(common.HexToHash("0x02"), common.HexToHash("0xaa"), common.HexToHash("0x01")) - defer it.Release() - verifyIterator(t, 3, it, verifyStorage) // expected: 01, 03, 05 - - it, _ = snaps.StorageIterator(common.HexToHash("0x02"), common.HexToHash("0xaa"), common.HexToHash("0x02")) - defer it.Release() - verifyIterator(t, 2, it, verifyStorage) // expected: 03, 05 - - it, _ = snaps.StorageIterator(common.HexToHash("0x02"), common.HexToHash("0xaa"), common.HexToHash("0x5")) - defer it.Release() - verifyIterator(t, 1, it, verifyStorage) // expected: 05 - - it, _ = snaps.StorageIterator(common.HexToHash("0x02"), common.HexToHash("0xaa"), common.HexToHash("0x6")) - defer it.Release() - verifyIterator(t, 0, it, verifyStorage) // expected: nothing - - it, _ = snaps.StorageIterator(common.HexToHash("0x04"), common.HexToHash("0xaa"), common.HexToHash("0x01")) - defer it.Release() - verifyIterator(t, 6, it, verifyStorage) // expected: 01, 02, 03, 05, 06, 08 - - it, _ = snaps.StorageIterator(common.HexToHash("0x04"), common.HexToHash("0xaa"), common.HexToHash("0x05")) - defer it.Release() - verifyIterator(t, 3, it, verifyStorage) // expected: 05, 06, 08 - - it, _ = snaps.StorageIterator(common.HexToHash("0x04"), common.HexToHash("0xaa"), common.HexToHash("0x08")) - defer it.Release() - verifyIterator(t, 1, it, verifyStorage) // expected: 08 - - it, _ = snaps.StorageIterator(common.HexToHash("0x04"), common.HexToHash("0xaa"), common.HexToHash("0x09")) - defer it.Release() - verifyIterator(t, 0, it, verifyStorage) // expected: nothing -} - -// TestAccountIteratorDeletions tests that the iterator behaves correct when there are -// deleted accounts (where the Account() value is nil). The iterator -// should not output any accounts or nil-values for those cases. -func TestAccountIteratorDeletions(t *testing.T) { - // Create an empty base layer and a snapshot tree out of it - base := &diskLayer{ - diskdb: rawdb.NewMemoryDatabase(), - root: common.HexToHash("0x01"), - cache: fastcache.New(1024 * 500), - } - snaps := &Tree{ - layers: map[common.Hash]snapshot{ - base.root: base, - }, - } - // Stack three diff layers on top with various overlaps - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), - nil, randomAccountSet("0x11", "0x22", "0x33"), nil) - - deleted := common.HexToHash("0x22") - destructed := map[common.Hash]struct{}{ - deleted: {}, - } - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), - destructed, randomAccountSet("0x11", "0x33"), nil) - - snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), - nil, randomAccountSet("0x33", "0x44", "0x55"), nil) - - // The output should be 11,33,44,55 - it, _ := snaps.AccountIterator(common.HexToHash("0x04"), common.Hash{}) - // Do a quick check - verifyIterator(t, 4, it, verifyAccount) - it.Release() - - // And a more detailed verification that we indeed do not see '0x22' - it, _ = snaps.AccountIterator(common.HexToHash("0x04"), common.Hash{}) - defer it.Release() - for it.Next() { - hash := it.Hash() - if it.Account() == nil { - t.Errorf("iterator returned nil-value for hash %x", hash) - } - if hash == deleted { - t.Errorf("expected deleted elem %x to not be returned by iterator", deleted) - } - } -} - -func TestStorageIteratorDeletions(t *testing.T) { - // Create an empty base layer and a snapshot tree out of it - base := &diskLayer{ - diskdb: rawdb.NewMemoryDatabase(), - root: common.HexToHash("0x01"), - cache: fastcache.New(1024 * 500), - } - snaps := &Tree{ - layers: map[common.Hash]snapshot{ - base.root: base, - }, - } - // Stack three diff layers on top with various overlaps - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, - randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x01", "0x03", "0x05"}}, nil)) - - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, - randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x02", "0x04", "0x06"}}, [][]string{{"0x01", "0x03"}})) - - // The output should be 02,04,05,06 - it, _ := snaps.StorageIterator(common.HexToHash("0x03"), common.HexToHash("0xaa"), common.Hash{}) - verifyIterator(t, 4, it, verifyStorage) - it.Release() - - // The output should be 04,05,06 - it, _ = snaps.StorageIterator(common.HexToHash("0x03"), common.HexToHash("0xaa"), common.HexToHash("0x03")) - verifyIterator(t, 3, it, verifyStorage) - it.Release() - - // Destruct the whole storage - destructed := map[common.Hash]struct{}{ - common.HexToHash("0xaa"): {}, - } - snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), destructed, nil, nil) - - it, _ = snaps.StorageIterator(common.HexToHash("0x04"), common.HexToHash("0xaa"), common.Hash{}) - verifyIterator(t, 0, it, verifyStorage) - it.Release() - - // Re-insert the slots of the same account - snaps.Update(common.HexToHash("0x05"), common.HexToHash("0x04"), nil, - randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x07", "0x08", "0x09"}}, nil)) - - // The output should be 07,08,09 - it, _ = snaps.StorageIterator(common.HexToHash("0x05"), common.HexToHash("0xaa"), common.Hash{}) - verifyIterator(t, 3, it, verifyStorage) - it.Release() - - // Destruct the whole storage but re-create the account in the same layer - snaps.Update(common.HexToHash("0x06"), common.HexToHash("0x05"), destructed, randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x11", "0x12"}}, nil)) - it, _ = snaps.StorageIterator(common.HexToHash("0x06"), common.HexToHash("0xaa"), common.Hash{}) - verifyIterator(t, 2, it, verifyStorage) // The output should be 11,12 - it.Release() - - verifyIterator(t, 2, snaps.Snapshot(common.HexToHash("0x06")).(*diffLayer).newBinaryStorageIterator(common.HexToHash("0xaa")), verifyStorage) -} - -// BenchmarkAccountIteratorTraversal is a bit a bit notorious -- all layers contain the -// exact same 200 accounts. That means that we need to process 2000 items, but -// only spit out 200 values eventually. -// -// The value-fetching benchmark is easy on the binary iterator, since it never has to reach -// down at any depth for retrieving the values -- all are on the toppmost layer -// -// BenchmarkAccountIteratorTraversal/binary_iterator_keys-6 2239 483674 ns/op -// BenchmarkAccountIteratorTraversal/binary_iterator_values-6 2403 501810 ns/op -// BenchmarkAccountIteratorTraversal/fast_iterator_keys-6 1923 677966 ns/op -// BenchmarkAccountIteratorTraversal/fast_iterator_values-6 1741 649967 ns/op -func BenchmarkAccountIteratorTraversal(b *testing.B) { - // Create a custom account factory to recreate the same addresses - makeAccounts := func(num int) map[common.Hash][]byte { - accounts := make(map[common.Hash][]byte) - for i := 0; i < num; i++ { - h := common.Hash{} - binary.BigEndian.PutUint64(h[:], uint64(i+1)) - accounts[h] = randomAccount() - } - return accounts - } - // Build up a large stack of snapshots - base := &diskLayer{ - diskdb: rawdb.NewMemoryDatabase(), - root: common.HexToHash("0x01"), - cache: fastcache.New(1024 * 500), - } - snaps := &Tree{ - layers: map[common.Hash]snapshot{ - base.root: base, - }, - } - for i := 1; i <= 100; i++ { - snaps.Update(common.HexToHash(fmt.Sprintf("0x%02x", i+1)), common.HexToHash(fmt.Sprintf("0x%02x", i)), nil, makeAccounts(200), nil) - } - // We call this once before the benchmark, so the creation of - // sorted accountlists are not included in the results. - head := snaps.Snapshot(common.HexToHash("0x65")) - head.(*diffLayer).newBinaryAccountIterator() - - b.Run("binary iterator keys", func(b *testing.B) { - for i := 0; i < b.N; i++ { - got := 0 - it := head.(*diffLayer).newBinaryAccountIterator() - for it.Next() { - got++ - } - if exp := 200; got != exp { - b.Errorf("iterator len wrong, expected %d, got %d", exp, got) - } - } - }) - b.Run("binary iterator values", func(b *testing.B) { - for i := 0; i < b.N; i++ { - got := 0 - it := head.(*diffLayer).newBinaryAccountIterator() - for it.Next() { - got++ - head.(*diffLayer).accountRLP(it.Hash(), 0) - } - if exp := 200; got != exp { - b.Errorf("iterator len wrong, expected %d, got %d", exp, got) - } - } - }) - b.Run("fast iterator keys", func(b *testing.B) { - for i := 0; i < b.N; i++ { - it, _ := snaps.AccountIterator(common.HexToHash("0x65"), common.Hash{}) - defer it.Release() - - got := 0 - for it.Next() { - got++ - } - if exp := 200; got != exp { - b.Errorf("iterator len wrong, expected %d, got %d", exp, got) - } - } - }) - b.Run("fast iterator values", func(b *testing.B) { - for i := 0; i < b.N; i++ { - it, _ := snaps.AccountIterator(common.HexToHash("0x65"), common.Hash{}) - defer it.Release() - - got := 0 - for it.Next() { - got++ - it.Account() - } - if exp := 200; got != exp { - b.Errorf("iterator len wrong, expected %d, got %d", exp, got) - } - } - }) -} - -// BenchmarkAccountIteratorLargeBaselayer is a pretty realistic benchmark, where -// the baselayer is a lot larger than the upper layer. -// -// This is heavy on the binary iterator, which in most cases will have to -// call recursively 100 times for the majority of the values -// -// BenchmarkAccountIteratorLargeBaselayer/binary_iterator_(keys)-6 514 1971999 ns/op -// BenchmarkAccountIteratorLargeBaselayer/binary_iterator_(values)-6 61 18997492 ns/op -// BenchmarkAccountIteratorLargeBaselayer/fast_iterator_(keys)-6 10000 114385 ns/op -// BenchmarkAccountIteratorLargeBaselayer/fast_iterator_(values)-6 4047 296823 ns/op -func BenchmarkAccountIteratorLargeBaselayer(b *testing.B) { - // Create a custom account factory to recreate the same addresses - makeAccounts := func(num int) map[common.Hash][]byte { - accounts := make(map[common.Hash][]byte) - for i := 0; i < num; i++ { - h := common.Hash{} - binary.BigEndian.PutUint64(h[:], uint64(i+1)) - accounts[h] = randomAccount() - } - return accounts - } - // Build up a large stack of snapshots - base := &diskLayer{ - diskdb: rawdb.NewMemoryDatabase(), - root: common.HexToHash("0x01"), - cache: fastcache.New(1024 * 500), - } - snaps := &Tree{ - layers: map[common.Hash]snapshot{ - base.root: base, - }, - } - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, makeAccounts(2000), nil) - for i := 2; i <= 100; i++ { - snaps.Update(common.HexToHash(fmt.Sprintf("0x%02x", i+1)), common.HexToHash(fmt.Sprintf("0x%02x", i)), nil, makeAccounts(20), nil) - } - // We call this once before the benchmark, so the creation of - // sorted accountlists are not included in the results. - head := snaps.Snapshot(common.HexToHash("0x65")) - head.(*diffLayer).newBinaryAccountIterator() - - b.Run("binary iterator (keys)", func(b *testing.B) { - for i := 0; i < b.N; i++ { - got := 0 - it := head.(*diffLayer).newBinaryAccountIterator() - for it.Next() { - got++ - } - if exp := 2000; got != exp { - b.Errorf("iterator len wrong, expected %d, got %d", exp, got) - } - } - }) - b.Run("binary iterator (values)", func(b *testing.B) { - for i := 0; i < b.N; i++ { - got := 0 - it := head.(*diffLayer).newBinaryAccountIterator() - for it.Next() { - got++ - v := it.Hash() - head.(*diffLayer).accountRLP(v, 0) - } - if exp := 2000; got != exp { - b.Errorf("iterator len wrong, expected %d, got %d", exp, got) - } - } - }) - b.Run("fast iterator (keys)", func(b *testing.B) { - for i := 0; i < b.N; i++ { - it, _ := snaps.AccountIterator(common.HexToHash("0x65"), common.Hash{}) - defer it.Release() - - got := 0 - for it.Next() { - got++ - } - if exp := 2000; got != exp { - b.Errorf("iterator len wrong, expected %d, got %d", exp, got) - } - } - }) - b.Run("fast iterator (values)", func(b *testing.B) { - for i := 0; i < b.N; i++ { - it, _ := snaps.AccountIterator(common.HexToHash("0x65"), common.Hash{}) - defer it.Release() - - got := 0 - for it.Next() { - it.Account() - got++ - } - if exp := 2000; got != exp { - b.Errorf("iterator len wrong, expected %d, got %d", exp, got) - } - } - }) -} - -/* -func BenchmarkBinaryAccountIteration(b *testing.B) { - benchmarkAccountIteration(b, func(snap snapshot) AccountIterator { - return snap.(*diffLayer).newBinaryAccountIterator() - }) -} - -func BenchmarkFastAccountIteration(b *testing.B) { - benchmarkAccountIteration(b, newFastAccountIterator) -} - -func benchmarkAccountIteration(b *testing.B, iterator func(snap snapshot) AccountIterator) { - // Create a diff stack and randomize the accounts across them - layers := make([]map[common.Hash][]byte, 128) - for i := 0; i < len(layers); i++ { - layers[i] = make(map[common.Hash][]byte) - } - for i := 0; i < b.N; i++ { - depth := rand.Intn(len(layers)) - layers[depth][randomHash()] = randomAccount() - } - stack := snapshot(emptyLayer()) - for _, layer := range layers { - stack = stack.Update(common.Hash{}, layer, nil, nil) - } - // Reset the timers and report all the stats - it := iterator(stack) - - b.ResetTimer() - b.ReportAllocs() - - for it.Next() { - } -} -*/ diff --git a/core/state/snapshot/snapshot_test.go b/core/state/snapshot/snapshot_test.go deleted file mode 100644 index 94e3610..0000000 --- a/core/state/snapshot/snapshot_test.go +++ /dev/null @@ -1,371 +0,0 @@ -// Copyright 2019 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package snapshot - -import ( - "fmt" - "math/big" - "math/rand" - "testing" - - "github.com/VictoriaMetrics/fastcache" - "github.com/ava-labs/coreth/core/rawdb" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/rlp" -) - -// randomHash generates a random blob of data and returns it as a hash. -func randomHash() common.Hash { - var hash common.Hash - if n, err := rand.Read(hash[:]); n != common.HashLength || err != nil { - panic(err) - } - return hash -} - -// randomAccount generates a random account and returns it RLP encoded. -func randomAccount() []byte { - root := randomHash() - a := Account{ - Balance: big.NewInt(rand.Int63()), - Nonce: rand.Uint64(), - Root: root[:], - CodeHash: emptyCode[:], - } - data, _ := rlp.EncodeToBytes(a) - return data -} - -// randomAccountSet generates a set of random accounts with the given strings as -// the account address hashes. -func randomAccountSet(hashes ...string) map[common.Hash][]byte { - accounts := make(map[common.Hash][]byte) - for _, hash := range hashes { - accounts[common.HexToHash(hash)] = randomAccount() - } - return accounts -} - -// randomStorageSet generates a set of random slots with the given strings as -// the slot addresses. -func randomStorageSet(accounts []string, hashes [][]string, nilStorage [][]string) map[common.Hash]map[common.Hash][]byte { - storages := make(map[common.Hash]map[common.Hash][]byte) - for index, account := range accounts { - storages[common.HexToHash(account)] = make(map[common.Hash][]byte) - - if index < len(hashes) { - hashes := hashes[index] - for _, hash := range hashes { - storages[common.HexToHash(account)][common.HexToHash(hash)] = randomHash().Bytes() - } - } - if index < len(nilStorage) { - nils := nilStorage[index] - for _, hash := range nils { - storages[common.HexToHash(account)][common.HexToHash(hash)] = nil - } - } - } - return storages -} - -// Tests that if a disk layer becomes stale, no active external references will -// be returned with junk data. This version of the test flattens every diff layer -// to check internal corner case around the bottom-most memory accumulator. -func TestDiskLayerExternalInvalidationFullFlatten(t *testing.T) { - // Create an empty base layer and a snapshot tree out of it - base := &diskLayer{ - diskdb: rawdb.NewMemoryDatabase(), - root: common.HexToHash("0x01"), - cache: fastcache.New(1024 * 500), - } - snaps := &Tree{ - layers: map[common.Hash]snapshot{ - base.root: base, - }, - } - // Retrieve a reference to the base and commit a diff on top - ref := snaps.Snapshot(base.root) - - accounts := map[common.Hash][]byte{ - common.HexToHash("0xa1"): randomAccount(), - } - if err := snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, accounts, nil); err != nil { - t.Fatalf("failed to create a diff layer: %v", err) - } - if n := len(snaps.layers); n != 2 { - t.Errorf("pre-cap layer count mismatch: have %d, want %d", n, 2) - } - // Commit the diff layer onto the disk and ensure it's persisted - if err := snaps.Cap(common.HexToHash("0x02"), 0); err != nil { - t.Fatalf("failed to merge diff layer onto disk: %v", err) - } - // Since the base layer was modified, ensure that data retrieval on the external reference fail - if acc, err := ref.Account(common.HexToHash("0x01")); err != ErrSnapshotStale { - t.Errorf("stale reference returned account: %#x (err: %v)", acc, err) - } - if slot, err := ref.Storage(common.HexToHash("0xa1"), common.HexToHash("0xb1")); err != ErrSnapshotStale { - t.Errorf("stale reference returned storage slot: %#x (err: %v)", slot, err) - } - if n := len(snaps.layers); n != 1 { - t.Errorf("post-cap layer count mismatch: have %d, want %d", n, 1) - fmt.Println(snaps.layers) - } -} - -// Tests that if a disk layer becomes stale, no active external references will -// be returned with junk data. This version of the test retains the bottom diff -// layer to check the usual mode of operation where the accumulator is retained. -func TestDiskLayerExternalInvalidationPartialFlatten(t *testing.T) { - // Create an empty base layer and a snapshot tree out of it - base := &diskLayer{ - diskdb: rawdb.NewMemoryDatabase(), - root: common.HexToHash("0x01"), - cache: fastcache.New(1024 * 500), - } - snaps := &Tree{ - layers: map[common.Hash]snapshot{ - base.root: base, - }, - } - // Retrieve a reference to the base and commit two diffs on top - ref := snaps.Snapshot(base.root) - - accounts := map[common.Hash][]byte{ - common.HexToHash("0xa1"): randomAccount(), - } - if err := snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, accounts, nil); err != nil { - t.Fatalf("failed to create a diff layer: %v", err) - } - if err := snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, accounts, nil); err != nil { - t.Fatalf("failed to create a diff layer: %v", err) - } - if n := len(snaps.layers); n != 3 { - t.Errorf("pre-cap layer count mismatch: have %d, want %d", n, 3) - } - // Commit the diff layer onto the disk and ensure it's persisted - defer func(memcap uint64) { aggregatorMemoryLimit = memcap }(aggregatorMemoryLimit) - aggregatorMemoryLimit = 0 - - if err := snaps.Cap(common.HexToHash("0x03"), 2); err != nil { - t.Fatalf("failed to merge diff layer onto disk: %v", err) - } - // Since the base layer was modified, ensure that data retrievald on the external reference fail - if acc, err := ref.Account(common.HexToHash("0x01")); err != ErrSnapshotStale { - t.Errorf("stale reference returned account: %#x (err: %v)", acc, err) - } - if slot, err := ref.Storage(common.HexToHash("0xa1"), common.HexToHash("0xb1")); err != ErrSnapshotStale { - t.Errorf("stale reference returned storage slot: %#x (err: %v)", slot, err) - } - if n := len(snaps.layers); n != 2 { - t.Errorf("post-cap layer count mismatch: have %d, want %d", n, 2) - fmt.Println(snaps.layers) - } -} - -// Tests that if a diff layer becomes stale, no active external references will -// be returned with junk data. This version of the test flattens every diff layer -// to check internal corner case around the bottom-most memory accumulator. -func TestDiffLayerExternalInvalidationFullFlatten(t *testing.T) { - // Create an empty base layer and a snapshot tree out of it - base := &diskLayer{ - diskdb: rawdb.NewMemoryDatabase(), - root: common.HexToHash("0x01"), - cache: fastcache.New(1024 * 500), - } - snaps := &Tree{ - layers: map[common.Hash]snapshot{ - base.root: base, - }, - } - // Commit two diffs on top and retrieve a reference to the bottommost - accounts := map[common.Hash][]byte{ - common.HexToHash("0xa1"): randomAccount(), - } - if err := snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, accounts, nil); err != nil { - t.Fatalf("failed to create a diff layer: %v", err) - } - if err := snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, accounts, nil); err != nil { - t.Fatalf("failed to create a diff layer: %v", err) - } - if n := len(snaps.layers); n != 3 { - t.Errorf("pre-cap layer count mismatch: have %d, want %d", n, 3) - } - ref := snaps.Snapshot(common.HexToHash("0x02")) - - // Flatten the diff layer into the bottom accumulator - if err := snaps.Cap(common.HexToHash("0x03"), 1); err != nil { - t.Fatalf("failed to flatten diff layer into accumulator: %v", err) - } - // Since the accumulator diff layer was modified, ensure that data retrievald on the external reference fail - if acc, err := ref.Account(common.HexToHash("0x01")); err != ErrSnapshotStale { - t.Errorf("stale reference returned account: %#x (err: %v)", acc, err) - } - if slot, err := ref.Storage(common.HexToHash("0xa1"), common.HexToHash("0xb1")); err != ErrSnapshotStale { - t.Errorf("stale reference returned storage slot: %#x (err: %v)", slot, err) - } - if n := len(snaps.layers); n != 2 { - t.Errorf("post-cap layer count mismatch: have %d, want %d", n, 2) - fmt.Println(snaps.layers) - } -} - -// Tests that if a diff layer becomes stale, no active external references will -// be returned with junk data. This version of the test retains the bottom diff -// layer to check the usual mode of operation where the accumulator is retained. -func TestDiffLayerExternalInvalidationPartialFlatten(t *testing.T) { - // Create an empty base layer and a snapshot tree out of it - base := &diskLayer{ - diskdb: rawdb.NewMemoryDatabase(), - root: common.HexToHash("0x01"), - cache: fastcache.New(1024 * 500), - } - snaps := &Tree{ - layers: map[common.Hash]snapshot{ - base.root: base, - }, - } - // Commit three diffs on top and retrieve a reference to the bottommost - accounts := map[common.Hash][]byte{ - common.HexToHash("0xa1"): randomAccount(), - } - if err := snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, accounts, nil); err != nil { - t.Fatalf("failed to create a diff layer: %v", err) - } - if err := snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, accounts, nil); err != nil { - t.Fatalf("failed to create a diff layer: %v", err) - } - if err := snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, accounts, nil); err != nil { - t.Fatalf("failed to create a diff layer: %v", err) - } - if n := len(snaps.layers); n != 4 { - t.Errorf("pre-cap layer count mismatch: have %d, want %d", n, 4) - } - ref := snaps.Snapshot(common.HexToHash("0x02")) - - // Doing a Cap operation with many allowed layers should be a no-op - exp := len(snaps.layers) - if err := snaps.Cap(common.HexToHash("0x04"), 2000); err != nil { - t.Fatalf("failed to flatten diff layer into accumulator: %v", err) - } - if got := len(snaps.layers); got != exp { - t.Errorf("layers modified, got %d exp %d", got, exp) - } - // Flatten the diff layer into the bottom accumulator - if err := snaps.Cap(common.HexToHash("0x04"), 2); err != nil { - t.Fatalf("failed to flatten diff layer into accumulator: %v", err) - } - // Since the accumulator diff layer was modified, ensure that data retrievald on the external reference fail - if acc, err := ref.Account(common.HexToHash("0x01")); err != ErrSnapshotStale { - t.Errorf("stale reference returned account: %#x (err: %v)", acc, err) - } - if slot, err := ref.Storage(common.HexToHash("0xa1"), common.HexToHash("0xb1")); err != ErrSnapshotStale { - t.Errorf("stale reference returned storage slot: %#x (err: %v)", slot, err) - } - if n := len(snaps.layers); n != 3 { - t.Errorf("post-cap layer count mismatch: have %d, want %d", n, 3) - fmt.Println(snaps.layers) - } -} - -// TestPostCapBasicDataAccess tests some functionality regarding capping/flattening. -func TestPostCapBasicDataAccess(t *testing.T) { - // setAccount is a helper to construct a random account entry and assign it to - // an account slot in a snapshot - setAccount := func(accKey string) map[common.Hash][]byte { - return map[common.Hash][]byte{ - common.HexToHash(accKey): randomAccount(), - } - } - // Create a starting base layer and a snapshot tree out of it - base := &diskLayer{ - diskdb: rawdb.NewMemoryDatabase(), - root: common.HexToHash("0x01"), - cache: fastcache.New(1024 * 500), - } - snaps := &Tree{ - layers: map[common.Hash]snapshot{ - base.root: base, - }, - } - // The lowest difflayer - snaps.Update(common.HexToHash("0xa1"), common.HexToHash("0x01"), nil, setAccount("0xa1"), nil) - snaps.Update(common.HexToHash("0xa2"), common.HexToHash("0xa1"), nil, setAccount("0xa2"), nil) - snaps.Update(common.HexToHash("0xb2"), common.HexToHash("0xa1"), nil, setAccount("0xb2"), nil) - - snaps.Update(common.HexToHash("0xa3"), common.HexToHash("0xa2"), nil, setAccount("0xa3"), nil) - snaps.Update(common.HexToHash("0xb3"), common.HexToHash("0xb2"), nil, setAccount("0xb3"), nil) - - // checkExist verifies if an account exiss in a snapshot - checkExist := func(layer *diffLayer, key string) error { - if data, _ := layer.Account(common.HexToHash(key)); data == nil { - return fmt.Errorf("expected %x to exist, got nil", common.HexToHash(key)) - } - return nil - } - // shouldErr checks that an account access errors as expected - shouldErr := func(layer *diffLayer, key string) error { - if data, err := layer.Account(common.HexToHash(key)); err == nil { - return fmt.Errorf("expected error, got data %x", data) - } - return nil - } - // check basics - snap := snaps.Snapshot(common.HexToHash("0xb3")).(*diffLayer) - - if err := checkExist(snap, "0xa1"); err != nil { - t.Error(err) - } - if err := checkExist(snap, "0xb2"); err != nil { - t.Error(err) - } - if err := checkExist(snap, "0xb3"); err != nil { - t.Error(err) - } - // Cap to a bad root should fail - if err := snaps.Cap(common.HexToHash("0x1337"), 0); err == nil { - t.Errorf("expected error, got none") - } - // Now, merge the a-chain - snaps.Cap(common.HexToHash("0xa3"), 0) - - // At this point, a2 got merged into a1. Thus, a1 is now modified, and as a1 is - // the parent of b2, b2 should no longer be able to iterate into parent. - - // These should still be accessible - if err := checkExist(snap, "0xb2"); err != nil { - t.Error(err) - } - if err := checkExist(snap, "0xb3"); err != nil { - t.Error(err) - } - // But these would need iteration into the modified parent - if err := shouldErr(snap, "0xa1"); err != nil { - t.Error(err) - } - if err := shouldErr(snap, "0xa2"); err != nil { - t.Error(err) - } - if err := shouldErr(snap, "0xa3"); err != nil { - t.Error(err) - } - // Now, merge it again, just for fun. It should now error, since a3 - // is a disk layer - if err := snaps.Cap(common.HexToHash("0xa3"), 0); err == nil { - t.Error("expected error capping the disk layer, got none") - } -} diff --git a/core/state/snapshot/wipe_test.go b/core/state/snapshot/wipe_test.go deleted file mode 100644 index a656982..0000000 --- a/core/state/snapshot/wipe_test.go +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright 2019 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package snapshot - -import ( - "math/rand" - "testing" - - "github.com/ava-labs/coreth/core/rawdb" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/ethdb/memorydb" -) - -// Tests that given a database with random data content, all parts of a snapshot -// can be crrectly wiped without touching anything else. -func TestWipe(t *testing.T) { - // Create a database with some random snapshot data - db := memorydb.New() - - for i := 0; i < 128; i++ { - account := randomHash() - rawdb.WriteAccountSnapshot(db, account, randomHash().Bytes()) - for j := 0; j < 1024; j++ { - rawdb.WriteStorageSnapshot(db, account, randomHash(), randomHash().Bytes()) - } - } - rawdb.WriteSnapshotRoot(db, randomHash()) - - // Add some random non-snapshot data too to make wiping harder - for i := 0; i < 65536; i++ { - // Generate a key that's the wrong length for a state snapshot item - var keysize int - for keysize == 0 || keysize == 32 || keysize == 64 { - keysize = 8 + rand.Intn(64) // +8 to ensure we will "never" randomize duplicates - } - // Randomize the suffix, dedup and inject it under the snapshot namespace - keysuffix := make([]byte, keysize) - rand.Read(keysuffix) - - if rand.Int31n(2) == 0 { - db.Put(append(rawdb.SnapshotAccountPrefix, keysuffix...), randomHash().Bytes()) - } else { - db.Put(append(rawdb.SnapshotStoragePrefix, keysuffix...), randomHash().Bytes()) - } - } - // Sanity check that all the keys are present - var items int - - it := db.NewIterator(rawdb.SnapshotAccountPrefix, nil) - defer it.Release() - - for it.Next() { - key := it.Key() - if len(key) == len(rawdb.SnapshotAccountPrefix)+common.HashLength { - items++ - } - } - it = db.NewIterator(rawdb.SnapshotStoragePrefix, nil) - defer it.Release() - - for it.Next() { - key := it.Key() - if len(key) == len(rawdb.SnapshotStoragePrefix)+2*common.HashLength { - items++ - } - } - if items != 128+128*1024 { - t.Fatalf("snapshot size mismatch: have %d, want %d", items, 128+128*1024) - } - if hash := rawdb.ReadSnapshotRoot(db); hash == (common.Hash{}) { - t.Errorf("snapshot block marker mismatch: have %#x, want ", hash) - } - // Wipe all snapshot entries from the database - <-wipeSnapshot(db, true) - - // Iterate over the database end ensure no snapshot information remains - it = db.NewIterator(rawdb.SnapshotAccountPrefix, nil) - defer it.Release() - - for it.Next() { - key := it.Key() - if len(key) == len(rawdb.SnapshotAccountPrefix)+common.HashLength { - t.Errorf("snapshot entry remained after wipe: %x", key) - } - } - it = db.NewIterator(rawdb.SnapshotStoragePrefix, nil) - defer it.Release() - - for it.Next() { - key := it.Key() - if len(key) == len(rawdb.SnapshotStoragePrefix)+2*common.HashLength { - t.Errorf("snapshot entry remained after wipe: %x", key) - } - } - if hash := rawdb.ReadSnapshotRoot(db); hash != (common.Hash{}) { - t.Errorf("snapshot block marker remained after wipe: %#x", hash) - } - // Iterate over the database and ensure miscellaneous items are present - items = 0 - - it = db.NewIterator(nil, nil) - defer it.Release() - - for it.Next() { - items++ - } - if items != 65536 { - t.Fatalf("misc item count mismatch: have %d, want %d", items, 65536) - } -} diff --git a/core/vm/instructions.go b/core/vm/instructions.go index 35ce39f..abfa2aa 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -17,6 +17,7 @@ package vm import ( + "errors" "github.com/ava-labs/coreth/core/types" "github.com/ava-labs/coreth/params" "github.com/ethereum/go-ethereum/common" @@ -265,7 +266,12 @@ func opBalance(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([ func opBalanceMultiCoin(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { addr, cid := callContext.stack.pop(), callContext.stack.pop() - callContext.stack.push(interpreter.evm.StateDB.GetBalanceMultiCoin(common.BigToAddress(addr), common.BigToHash(cid))) + res, err := uint256.FromBig(interpreter.evm.StateDB.GetBalanceMultiCoin( + common.BigToAddress(addr.ToBig()), common.BigToHash(cid.ToBig()))) + if err { + return nil, errors.New("balance overflow") + } + callContext.stack.push(res) return nil, nil } @@ -725,7 +731,7 @@ func opCallExpert(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) // Pop other call parameters. addr, value, cid, value2, inOffset, inSize, retOffset, retSize := stack.pop(), stack.pop(), stack.pop(), stack.pop(), stack.pop(), stack.pop(), stack.pop(), stack.pop() toAddr := common.Address(addr.Bytes20()) - coinID := common.BigToHash(cid) + coinID := common.BigToHash(cid.ToBig()) // Get the arguments from the memory. args := callContext.memory.GetPtr(int64(inOffset.Uint64()), int64(inSize.Uint64())) @@ -875,8 +881,8 @@ func opSuicide(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([ return nil, nil } -func opEMC(pc *uint64, interpreter *EVMInterpreter, contract *Contract, callContext *callCtx) ([]byte, error) { - return nil, interpreter.evm.StateDB.EnableMultiCoin(contract.Address()) +func opEMC(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { + return nil, interpreter.evm.StateDB.EnableMultiCoin(callContext.contract.Address()) } // following functions are used by the instruction jump table diff --git a/coreth.go b/coreth.go index 1fe2613..4d0c2ee 100644 --- a/coreth.go +++ b/coreth.go @@ -2,7 +2,7 @@ package coreth import ( "crypto/ecdsa" - "fmt" + //"fmt" "io" "os" @@ -17,7 +17,8 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/trie" + //"github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/mattn/go-isatty" ) @@ -52,18 +53,18 @@ func NewETHChain(config *eth.Config, nodecfg *node.Config, etherBase *common.Add if nodecfg == nil { nodecfg = &node.Config{} } - mux := new(event.TypeMux) - ctx, ep, err := node.NewServiceContext(nodecfg, mux) + //mux := new(event.TypeMux) + node, err := node.New(nodecfg) if err != nil { panic(err) } - if ep != "" { - log.Info(fmt.Sprintf("temporary keystore = %s", ep)) - } + //if ep != "" { + // log.Info(fmt.Sprintf("temporary keystore = %s", ep)) + //} cb := new(dummy.ConsensusCallbacks) mcb := new(miner.MinerCallbacks) bcb := new(eth.BackendCallbacks) - backend, _ := eth.New(&ctx, config, cb, mcb, bcb, chainDB) + backend, _ := eth.New(node, config, cb, mcb, bcb, chainDB) chain := ÐChain{backend: backend, cb: cb, mcb: mcb, bcb: bcb} if etherBase == nil { etherBase = &BlackholeAddr @@ -85,7 +86,7 @@ func (self *ETHChain) GenBlock() { } func (self *ETHChain) VerifyBlock(block *types.Block) bool { - txnHash := types.DeriveSha(block.Transactions()) + txnHash := types.DeriveSha(block.Transactions(), new(trie.Trie)) uncleHash := types.CalcUncleHash(block.Uncles()) ethHeader := block.Header() if txnHash != ethHeader.TxHash || uncleHash != ethHeader.UncleHash { diff --git a/eth/api.go b/eth/api.go index 92ac928..ada2a97 100644 --- a/eth/api.go +++ b/eth/api.go @@ -371,7 +371,7 @@ func (api *PublicDebugAPI) AccountRange(blockNrOrHash rpc.BlockNumberOrHash, sta var block *types.Block if number == rpc.LatestBlockNumber { block = api.eth.blockchain.CurrentBlock() - } else if blockNr == rpc.AcceptedBlockNumber { + } else if number == rpc.AcceptedBlockNumber { block = api.eth.AcceptedBlock() } else { block = api.eth.blockchain.GetBlockByNumber(uint64(number)) diff --git a/eth/backend.go b/eth/backend.go index 9222181..b1c29a5 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -41,7 +41,6 @@ import ( "github.com/ava-labs/coreth/node" "github.com/ava-labs/coreth/params" "github.com/ava-labs/coreth/rpc" - "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/bloombits" @@ -50,6 +49,7 @@ import ( "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/rlp" ) @@ -127,7 +127,7 @@ func New(stack *node.Node, config *Config, var err error if chainDb == nil { // Assemble the Ethereum object - chainDb, err = ctx.OpenDatabaseWithFreezer("chaindata", config.DatabaseCache, config.DatabaseHandles, config.DatabaseFreezer, "eth/db/chaindata/") + chainDb, err = stack.OpenDatabaseWithFreezer("chaindata", config.DatabaseCache, config.DatabaseHandles, config.DatabaseFreezer, "eth/db/chaindata/") if err != nil { return nil, err } @@ -223,7 +223,7 @@ func New(stack *node.Node, config *Config, } eth.APIBackend.gpo = gasprice.NewOracle(eth.APIBackend, gpoParams) - eth.dialCandidates, err = eth.setupDiscovery(&stack.Config().P2P) + //eth.dialCandidates, err = eth.setupDiscovery(&stack.Config().P2P) if err != nil { return nil, err } @@ -233,8 +233,8 @@ func New(stack *node.Node, config *Config, // Register the backend on the node stack.RegisterAPIs(eth.APIs()) - stack.RegisterProtocols(eth.Protocols()) - stack.RegisterLifecycle(eth) + //stack.RegisterProtocols(eth.Protocols()) + //stack.RegisterLifecycle(eth) return eth, nil } diff --git a/eth/protocol.go b/eth/protocol.go index 94260c2..ef5dcde 100644 --- a/eth/protocol.go +++ b/eth/protocol.go @@ -22,6 +22,7 @@ import ( "math/big" "github.com/ava-labs/coreth/core" + "github.com/ava-labs/coreth/core/forkid" "github.com/ava-labs/coreth/core/types" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/event" diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 8753c05..9915ad4 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -30,7 +30,6 @@ import ( "github.com/ava-labs/coreth/accounts/scwallet" "github.com/ava-labs/coreth/consensus/ethash" "github.com/ava-labs/coreth/core" - "github.com/ava-labs/coreth/core/rawdb" "github.com/ava-labs/coreth/core/types" "github.com/ava-labs/coreth/core/vm" "github.com/ava-labs/coreth/params" diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index ae4196f..cee6b57 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -22,6 +22,7 @@ import ( "math/big" "github.com/ava-labs/coreth/accounts" + "github.com/ava-labs/coreth/consensus" "github.com/ava-labs/coreth/core" "github.com/ava-labs/coreth/core/state" "github.com/ava-labs/coreth/core/types" diff --git a/miner/miner.go b/miner/miner.go index 57938e1..e8e59a4 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -30,7 +30,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/log" ) // Backend wraps all methods required for mining. @@ -60,7 +59,7 @@ type Miner struct { func New(eth Backend, config *Config, chainConfig *params.ChainConfig, mux *event.TypeMux, engine consensus.Engine, isLocalBlock func(block *types.Block) bool, mcb *MinerCallbacks) *Miner { return &Miner{ - worker: newWorker(config, chainConfig, engine, eth, mux, isLocalBlock, mcb), + worker: newWorker(config, chainConfig, engine, eth, mux, isLocalBlock, true, mcb), } } diff --git a/node/api.go b/node/api.go index e74d10b..4589d25 100644 --- a/node/api.go +++ b/node/api.go @@ -21,6 +21,7 @@ import ( "fmt" //"strings" + "github.com/ava-labs/coreth/internal/debug" "github.com/ava-labs/coreth/rpc" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/crypto" diff --git a/node/node.go b/node/node.go index c6e4181..3ed89ed 100644 --- a/node/node.go +++ b/node/node.go @@ -18,14 +18,13 @@ package node import ( "errors" + "os" "path/filepath" - "reflect" "strings" "sync" "github.com/ava-labs/coreth/accounts" "github.com/ava-labs/coreth/core/rawdb" - "github.com/ava-labs/coreth/internal/debug" "github.com/ava-labs/coreth/rpc" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -48,7 +47,6 @@ type Node struct { state int // Tracks state of node lifecycle lock sync.Mutex - lifecycles []Lifecycle // All registered backends, services, and auxiliary services that have a lifecycle rpcAPIs []rpc.API // List of APIs currently provided by the node inprocHandler *rpc.Server // In-process RPC request handler to process the API requests @@ -61,6 +59,25 @@ const ( closedState ) +func (n *Node) openDataDir() error { + if n.config.DataDir == "" { + return nil // ephemeral + } + + instdir := filepath.Join(n.config.DataDir, n.config.name()) + if err := os.MkdirAll(instdir, 0700); err != nil { + return err + } + // Lock the instance directory to prevent concurrent use by another instance as well as + // accidental use of the instance directory as a database. + release, _, err := fileutil.Flock(filepath.Join(instdir, "LOCK")) + if err != nil { + return convertFileLockError(err) + } + n.dirLock = release + return nil +} + // New creates a new P2P node, ready for protocol registration. func New(conf *Config) (*Node, error) { // Copy config and resolve the datadir so future changes to the current @@ -266,3 +283,14 @@ func (n *Node) closeDatabases() (errors []error) { } return errors } + +// RegisterAPIs registers the APIs a service provides on the node. +func (n *Node) RegisterAPIs(apis []rpc.API) { + n.lock.Lock() + defer n.lock.Unlock() + + if n.state != initializingState { + panic("can't register APIs on running/stopped node") + } + n.rpcAPIs = append(n.rpcAPIs, apis...) +} diff --git a/rpc/metrics.go b/rpc/metrics.go new file mode 100644 index 0000000..7fb6fc0 --- /dev/null +++ b/rpc/metrics.go @@ -0,0 +1,39 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rpc + +import ( + "fmt" + + "github.com/ethereum/go-ethereum/metrics" +) + +var ( + rpcRequestGauge = metrics.NewRegisteredGauge("rpc/requests", nil) + successfulRequestGauge = metrics.NewRegisteredGauge("rpc/success", nil) + failedReqeustGauge = metrics.NewRegisteredGauge("rpc/failure", nil) + rpcServingTimer = metrics.NewRegisteredTimer("rpc/duration/all", nil) +) + +func newRPCServingTimer(method string, valid bool) metrics.Timer { + flag := "success" + if !valid { + flag = "failure" + } + m := fmt.Sprintf("rpc/duration/%s/%s", method, flag) + return metrics.GetOrRegisterTimer(m, nil) +} -- cgit v1.2.3-70-g09d2