Merge branch 'master' into pmikolajczyk/nit-3121-uncompressed-batch-size-limit

Author: pmikolajczyk41

arbitrum/apibackend.go

@@ -545,6 +545,21 @@ func StateAndHeaderFromHeader(ctx context.Context, chainDb ethdb.Database, bc *c
 	if archiveClientsManager != nil && header.Number.Uint64() <= archiveClientsManager.lastAvailableBlock() {
 		return nil, header, &types.ErrUseArchiveFallback{BlockNum: header.Number.Uint64()}
 	}
+
+	// use upstream path for PathScheme, as:
+	// - intermediate state recreation and trie node referencing doesn't apply to pathdb
+	// - HistoricState is supported only by pathdb
+	if bc.TrieDB().Scheme() == rawdb.PathScheme {
+		statedb, err := bc.StateAt(header.Root)
+		if err != nil {
+			statedb, err = bc.HistoricState(header.Root)
+			if err != nil {
+				return nil, nil, err
+			}
+		}
+		return statedb, header, nil
+	}
+
 	stateFor := func(db state.Database, snapshots *snapshot.Tree) func(header *types.Header) (*state.StateDB, StateReleaseFunc, error) {
 		return func(header *types.Header) (*state.StateDB, StateReleaseFunc, error) {
 			if header.Root != (common.Hash{}) {

arbitrum/backend.go

@@ -41,7 +41,7 @@ type Backend struct {
 	filterSystem *filters.FilterSystem
 }
 
-func NewBackend(stack *node.Node, config *Config, chainDb ethdb.Database, publisher ArbInterface, filterConfig filters.Config) (*Backend, *filters.FilterSystem, error) {
+func NewBackend(stack *node.Node, config *Config, chainDb ethdb.Database, publisher ArbInterface, filterConfig filters.Config, stateScheme string) (*Backend, *filters.FilterSystem, error) {
 	backend := &Backend{
 		arb:     publisher,
 		stack:   stack,
@@ -55,7 +55,7 @@ func NewBackend(stack *node.Node, config *Config, chainDb ethdb.Database, publis
 		chanNewBlock: make(chan struct{}, 1),
 	}
 
-	scheme, err := rawdb.ParseStateScheme(config.StateScheme, chainDb)
+	scheme, err := rawdb.ParseStateScheme(stateScheme, chainDb)
 	if err != nil {
 		return nil, nil, err
 	}

arbitrum/config.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/eth/ethconfig"
 	flag "github.com/spf13/pflag"
 )
@@ -27,11 +26,6 @@ type Config struct {
 	LogNoHistory         bool   `koanf:"log-no-history"`         // No log search index is maintained.
 	LogExportCheckpoints string `koanf:"log-export-checkpoints"` // export log index checkpoints to file
 
-	// State scheme represents the scheme used to store states and trie
-	// nodes on top. It can be 'hash', 'path', or none which means use the scheme
-	// consistent with persistent state.
-	StateScheme string `koanf:"state-scheme"`
-
 	// Parameters for the filter system
 	FilterLogCacheSize int           `koanf:"filter-log-cache-size"`
 	FilterTimeout      time.Duration `koanf:"filter-timeout"`
@@ -82,7 +76,6 @@ func ConfigAddOptions(prefix string, f *flag.FlagSet) {
 	f.Uint64(prefix+".log-history", DefaultConfig.LogHistory, "maximum number of blocks from head where a log search index is maintained")
 	f.Bool(prefix+".log-no-history", DefaultConfig.LogNoHistory, "no log search index is maintained")
 	f.String(prefix+".log-export-checkpoints", DefaultConfig.LogExportCheckpoints, "export log index checkpoints to file")
-	f.String(prefix+".state-scheme", DefaultConfig.StateScheme, "state scheme used to store states and trie nodes on top")
 	f.Uint64(prefix+".feehistory-max-block-count", DefaultConfig.FeeHistoryMaxBlockCount, "max number of blocks a fee history request may cover")
 	f.String(prefix+".classic-redirect", DefaultConfig.ClassicRedirect, "url to redirect classic requests, use \"error:[CODE:]MESSAGE\" to return specified error instead of redirecting")
 	f.Duration(prefix+".classic-redirect-timeout", DefaultConfig.ClassicRedirectTimeout, "timeout for forwarded classic requests, where 0 = no timeout")
@@ -120,5 +113,4 @@ var DefaultConfig = Config{
 		TimeoutQueueBound: 512,
 	},
 	BlockRedirectsList: "default",
-	StateScheme:        rawdb.HashScheme,
 }

core/blockchain.go

@@ -172,9 +172,10 @@ type BlockChainConfig struct {
 	TrieNoAsyncFlush     bool          // Whether the asynchronous buffer flushing is disallowed
 	TrieJournalDirectory string        // Directory path to the journal used for persisting trie data across node restarts
 
-	Preimages   bool   // Whether to store preimage of trie key to the disk
-	StateScheme string // Scheme used to store ethereum states and merkle tree nodes on top
-	ArchiveMode bool   // Whether to enable the archive mode
+	Preimages     bool   // Whether to store preimage of trie key to the disk
+	StateScheme   string // Scheme used to store ethereum states and merkle tree nodes on top
+	ArchiveMode   bool   // Whether to enable the archive mode
+	MaxDiffLayers int    // MaxDiffLayers is the maximum diff layers allowed in the layer tree. (only for path scheme)
 
 	// Number of blocks from the chain head for which state histories are retained.
 	// If set to 0, all state histories across the entire chain will be retained;
@@ -228,6 +229,7 @@ func DefaultConfig() *BlockChainConfig {
 		TrieTimeLimitRandomOffset:          0,
 		MaxNumberOfBlocksToSkipStateSaving: 0,
 		MaxAmountOfGasToSkipStateSaving:    0,
+		MaxDiffLayers:                      pathdb.DefaultMaxDiffLayers,
 
 		TrieCleanLimit:   256,
 		TrieDirtyLimit:   256,
@@ -283,6 +285,7 @@ func (cfg *BlockChainConfig) triedbConfig(isVerkle bool) *triedb.Config {
 			TrieCleanSize:       cfg.TrieCleanLimit * 1024 * 1024,
 			StateCleanSize:      cfg.SnapshotLimit * 1024 * 1024,
 			JournalDirectory:    cfg.TrieJournalDirectory,
+			MaxDiffLayers:       cfg.MaxDiffLayers,
 
 			// TODO(rjl493456442): The write buffer represents the memory limit used
 			// for flushing both trie data and state data to disk. The config name

eth/backend.go

@@ -235,6 +235,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 			SnapshotLimit:    config.SnapshotCache,
 			Preimages:        config.Preimages,
 			StateHistory:     config.StateHistory,
+			MaxDiffLayers:    config.MaxDiffLayers,
 			StateScheme:      scheme,
 			ChainHistoryMode: config.HistoryMode,
 			TxIndexer: &core.TxIndexerConfig{

eth/ethconfig/config.go

@@ -35,6 +35,7 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/miner"
 	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/triedb/pathdb"
 )
 
 // FullNodeGPO contains default gasprice oracle settings for full node.
@@ -56,6 +57,7 @@ var Defaults = Config{
 	TransactionHistory: 2350000,
 	LogHistory:         2350000,
 	StateHistory:       params.FullImmutabilityThreshold,
+	MaxDiffLayers:      pathdb.DefaultMaxDiffLayers,
 	DatabaseCache:      512,
 	TrieCleanCache:     154,
 	TrieDirtyCache:     256,
@@ -105,6 +107,7 @@ type Config struct {
 	LogNoHistory         bool   `toml:",omitempty"` // No log search index is maintained.
 	LogExportCheckpoints string // export log index checkpoints to file
 	StateHistory         uint64 `toml:",omitempty"` // The maximum number of blocks from head whose state histories are reserved.
+	MaxDiffLayers        int    `toml:",omitempty"` // Maximum diff layers allowed in the layer tree.
 
 	// State scheme represents the scheme used to store ethereum states and trie
 	// nodes on top. It can be 'hash', 'path', or none which means use the scheme

triedb/pathdb/config.go

@@ -43,11 +43,9 @@ const (
 	// Do not increase the buffer size arbitrarily, otherwise the system
 	// pause time will increase when the database writes happen.
 	defaultBufferSize = 64 * 1024 * 1024
-)
 
-var (
-	// maxDiffLayers is the maximum diff layers allowed in the layer tree.
-	maxDiffLayers = 128
+	// Maximum diff layers allowed in the layer tree.
+	DefaultMaxDiffLayers = 128
 )
 
 // Defaults contains default settings for Ethereum mainnet.
@@ -57,18 +55,21 @@ var Defaults = &Config{
 	TrieCleanSize:       defaultTrieCleanSize,
 	StateCleanSize:      defaultStateCleanSize,
 	WriteBufferSize:     defaultBufferSize,
+	MaxDiffLayers:       DefaultMaxDiffLayers,
 }
 
 // ReadOnly is the config in order to open database in read only mode.
 var ReadOnly = &Config{
 	ReadOnly:       true,
 	TrieCleanSize:  defaultTrieCleanSize,
 	StateCleanSize: defaultStateCleanSize,
+	MaxDiffLayers:  DefaultMaxDiffLayers,
 }
 
 // Config contains the settings for database.
 type Config struct {
 	StateHistory        uint64 // Number of recent blocks to maintain state history for, 0: full chain
+	MaxDiffLayers       int    // Maximum diff layers allowed in the layer tree.
 	EnableStateIndexing bool   // Whether to enable state history indexing for external state access
 	TrieCleanSize       int    // Maximum memory allowance (in bytes) for caching clean trie data
 	StateCleanSize      int    // Maximum memory allowance (in bytes) for caching clean state data
@@ -90,6 +91,11 @@ func (c *Config) sanitize() *Config {
 		log.Warn("Sanitizing invalid node buffer size", "provided", common.StorageSize(conf.WriteBufferSize), "updated", common.StorageSize(maxBufferSize))
 		conf.WriteBufferSize = maxBufferSize
 	}
+
+	if conf.MaxDiffLayers <= 0 {
+		log.Warn("Sanitizing invalid max diff layers", "provided", conf.MaxDiffLayers, "updated", DefaultMaxDiffLayers)
+		conf.MaxDiffLayers = DefaultMaxDiffLayers
+	}
 	return &conf
 }
 

triedb/pathdb/database.go

@@ -342,7 +342,7 @@ func (db *Database) Update(root common.Hash, parentRoot common.Hash, block uint6
 	// - head-1 layer is paired with HEAD-1 state
 	// - head-127 layer(bottom-most diff layer) is paired with HEAD-127 state
 	// - head-128 layer(disk layer) is paired with HEAD-128 state
-	return db.tree.cap(root, maxDiffLayers)
+	return db.tree.cap(root, db.config.MaxDiffLayers)
 }
 
 // Commit traverses downwards the layer tree from a specified layer with the

triedb/pathdb/database_test.go

@@ -139,11 +139,12 @@ type tester struct {
 
 // testerConfig holds configuration parameters for running a test scenario.
 type testerConfig struct {
-	stateHistory uint64 // Number of historical states to retain
-	layers       int    // Number of state transitions to generate for
-	enableIndex  bool   // Enable state history indexing or not
-	journalDir   string // Directory path for persisting journal files
-	isVerkle     bool   // Enables Verkle trie mode if true
+	stateHistory  uint64 // Number of historical states to retain
+	maxDiffLayers int    // Maximum number of diff layers to maintain
+	layers        int    // Number of state transitions to generate for
+	enableIndex   bool   // Enable state history indexing or not
+	journalDir    string // Directory path for persisting journal files
+	isVerkle      bool   // Enables Verkle trie mode if true
 
 	writeBuffer *int // Optional, the size of memory allocated for write buffer
 	trieCache   *int // Optional, the size of memory allocated for trie cache
@@ -177,6 +178,7 @@ func newTester(t *testing.T, config *testerConfig) *tester {
 		db      = New(disk, &Config{
 			StateHistory:        config.stateHistory,
 			EnableStateIndexing: config.enableIndex,
+			MaxDiffLayers:       config.maxDiffLayers,
 			TrieCleanSize:       config.trieCacheSize(),
 			StateCleanSize:      config.stateCacheSize(),
 			WriteBufferSize:     config.writeBufferSize(),
@@ -579,13 +581,7 @@ func (t *tester) bottomIndex() int {
 }
 
 func TestDatabaseRollback(t *testing.T) {
-	// Redefine the diff layer depth allowance for faster testing.
-	maxDiffLayers = 4
-	defer func() {
-		maxDiffLayers = 128
-	}()
-
-	tester := newTester(t, &testerConfig{layers: 32})
+	tester := newTester(t, &testerConfig{layers: 32, maxDiffLayers: 4})
 	defer tester.release()
 
 	if err := tester.verifyHistory(); err != nil {
@@ -612,14 +608,8 @@ func TestDatabaseRollback(t *testing.T) {
 }
 
 func TestDatabaseRecoverable(t *testing.T) {
-	// Redefine the diff layer depth allowance for faster testing.
-	maxDiffLayers = 4
-	defer func() {
-		maxDiffLayers = 128
-	}()
-
 	var (
-		tester = newTester(t, &testerConfig{layers: 12})
+		tester = newTester(t, &testerConfig{layers: 12, maxDiffLayers: 4})
 		index  = tester.bottomIndex()
 	)
 	defer tester.release()
@@ -657,13 +647,7 @@ func TestDatabaseRecoverable(t *testing.T) {
 }
 
 func TestExecuteRollback(t *testing.T) {
-	// Redefine the diff layer depth allowance for faster testing.
-	maxDiffLayers = 4
-	defer func() {
-		maxDiffLayers = 128
-	}()
-
-	tester := newTester(t, &testerConfig{layers: 32})
+	tester := newTester(t, &testerConfig{layers: 32, maxDiffLayers: 4})
 	defer tester.release()
 
 	// Revert database from top to bottom
@@ -710,13 +694,7 @@ func TestExecuteRollback(t *testing.T) {
 }
 
 func TestDisable(t *testing.T) {
-	// Redefine the diff layer depth allowance for faster testing.
-	maxDiffLayers = 4
-	defer func() {
-		maxDiffLayers = 128
-	}()
-
-	tester := newTester(t, &testerConfig{layers: 32})
+	tester := newTester(t, &testerConfig{layers: 32, maxDiffLayers: 4})
 	defer tester.release()
 
 	stored := crypto.Keccak256Hash(rawdb.ReadAccountTrieNode(tester.db.diskdb, nil))
@@ -748,13 +726,7 @@ func TestDisable(t *testing.T) {
 }
 
 func TestCommit(t *testing.T) {
-	// Redefine the diff layer depth allowance for faster testing.
-	maxDiffLayers = 4
-	defer func() {
-		maxDiffLayers = 128
-	}()
-
-	tester := newTester(t, &testerConfig{layers: 12})
+	tester := newTester(t, &testerConfig{layers: 12, maxDiffLayers: 4})
 	defer tester.release()
 
 	if err := tester.db.Commit(tester.lastHash(), false); err != nil {
@@ -783,13 +755,7 @@ func TestJournal(t *testing.T) {
 }
 
 func testJournal(t *testing.T, journalDir string) {
-	// Redefine the diff layer depth allowance for faster testing.
-	maxDiffLayers = 4
-	defer func() {
-		maxDiffLayers = 128
-	}()
-
-	tester := newTester(t, &testerConfig{layers: 12, journalDir: journalDir})
+	tester := newTester(t, &testerConfig{layers: 12, maxDiffLayers: 4, journalDir: journalDir})
 	defer tester.release()
 
 	if err := tester.db.Journal(tester.lastHash()); err != nil {
@@ -830,13 +796,7 @@ func TestCorruptedJournal(t *testing.T) {
 }
 
 func testCorruptedJournal(t *testing.T, journalDir string, modifyFn func(database ethdb.Database)) {
-	// Redefine the diff layer depth allowance for faster testing.
-	maxDiffLayers = 4
-	defer func() {
-		maxDiffLayers = 128
-	}()
-
-	tester := newTester(t, &testerConfig{layers: 12, journalDir: journalDir})
+	tester := newTester(t, &testerConfig{layers: 12, maxDiffLayers: 4, journalDir: journalDir})
 	defer tester.release()
 
 	if err := tester.db.Journal(tester.lastHash()); err != nil {
@@ -875,13 +835,7 @@ func testCorruptedJournal(t *testing.T, journalDir string, modifyFn func(databas
 // truncating the tail histories. This ensures that the ID of the persistent state
 // always falls within the range of [oldest-history-id, latest-history-id].
 func TestTailTruncateHistory(t *testing.T) {
-	// Redefine the diff layer depth allowance for faster testing.
-	maxDiffLayers = 4
-	defer func() {
-		maxDiffLayers = 128
-	}()
-
-	tester := newTester(t, &testerConfig{layers: 12, stateHistory: 10})
+	tester := newTester(t, &testerConfig{layers: 12, maxDiffLayers: 4, stateHistory: 10})
 	defer tester.release()
 
 	tester.db.Close()
@@ -919,17 +873,13 @@ func copyStorages(set map[common.Hash]map[common.Hash][]byte) map[common.Hash]ma
 }
 
 func TestDatabaseIndexRecovery(t *testing.T) {
-	maxDiffLayers = 4
-	defer func() {
-		maxDiffLayers = 128
-	}()
-
 	//log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelDebug, true)))
 	writeBuffer := 512 * 1024
 	config := &testerConfig{
-		layers:      64,
-		enableIndex: true,
-		writeBuffer: &writeBuffer,
+		layers:        64,
+		maxDiffLayers: 4,
+		enableIndex:   true,
+		writeBuffer:   &writeBuffer,
 	}
 	env := newTester(t, config)
 	defer env.release()