From 34874c8899042e01ef8d1997a9b2dbf1dbbdcab8 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Fri, 19 Feb 2021 10:30:30 -0800 Subject: [PATCH 01/17] Add eth_sendBundle RPC --- core/mev_bundle.go | 30 ++++++++++++++++++ core/tx_pool.go | 63 ++++++++++++++++++++++++++++++++++--- core/tx_pool_test.go | 10 ++++++ eth/api_backend.go | 4 +++ internal/ethapi/api.go | 36 +++++++++++++++++++++ internal/ethapi/backend.go | 6 ++++ internal/web3ext/web3ext.go | 5 +++ les/api_backend.go | 3 ++ light/txpool.go | 11 +++++++ 9 files changed, 163 insertions(+), 5 deletions(-) create mode 100644 core/mev_bundle.go diff --git a/core/mev_bundle.go b/core/mev_bundle.go new file mode 100644 index 000000000000..257411708e74 --- /dev/null +++ b/core/mev_bundle.go @@ -0,0 +1,30 @@ +// Copyright 2014 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package core + +import ( + "math/big" + + "github.com/ethereum/go-ethereum/core/types" +) + +type mevBundle struct { + txs types.Transactions + blockNumber *big.Int + minTimestamp uint64 + maxTimestamp uint64 +} diff --git a/core/tx_pool.go b/core/tx_pool.go index c5b6047486e9..592d9bddf9b9 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -242,11 +242,12 @@ type TxPool struct { locals *accountSet // Set of local transaction to exempt from eviction rules journal *txJournal // Journal of local transaction to back up to disk - pending map[common.Address]*txList // All currently processable transactions - queue map[common.Address]*txList // Queued but non-processable transactions - beats map[common.Address]time.Time // Last heartbeat from each known account - all *txLookup // All transactions to allow lookups - priced *txPricedList // All transactions sorted by price + pending map[common.Address]*txList // All currently processable transactions + queue map[common.Address]*txList // Queued but non-processable transactions + beats map[common.Address]time.Time // Last heartbeat from each known account + mevBundles []mevBundle + all *txLookup // All transactions to allow lookups + priced *txPricedList // All transactions sorted by price chainHeadCh chan ChainHeadEvent chainHeadSub event.Subscription @@ -542,6 +543,58 @@ func (pool *TxPool) Pending(enforceTips bool) (map[common.Address]types.Transact return pending, nil } +/// AllMevBundles returns all the MEV Bundles currently in the pool +func (pool *TxPool) AllMevBundles() []mevBundle { + return pool.mevBundles +} + +// MevBundles returns a list of bundles valid for the given blockNumber/blockTimestamp +// also prunes bundles that are outdated +func (pool *TxPool) MevBundles(blockNumber *big.Int, blockTimestamp uint64) ([]types.Transactions, error) { + pool.mu.Lock() + defer pool.mu.Unlock() + + // returned values + var txBundles []types.Transactions + // rolled over values + var bundles []mevBundle + + for _, bundle := range pool.mevBundles { + // Prune outdated bundles + if (bundle.maxTimestamp != 0 && blockTimestamp > bundle.maxTimestamp) || blockNumber.Cmp(bundle.blockNumber) > 0 { + continue + } + + // Roll over future bundles + if (bundle.minTimestamp != 0 && blockTimestamp < bundle.minTimestamp) || blockNumber.Cmp(bundle.blockNumber) < 0 { + bundles = append(bundles, bundle) + continue + } + + // return the ones which are in time + txBundles = append(txBundles, bundle.txs) + // keep the bundles around internally until they need to be pruned + bundles = append(bundles, bundle) + } + + pool.mevBundles = bundles + return txBundles, nil +} + +// AddMevBundle adds a mev bundle to the pool +func (pool *TxPool) AddMevBundle(txs types.Transactions, blockNumber *big.Int, minTimestamp, maxTimestamp uint64) error { + pool.mu.Lock() + defer pool.mu.Unlock() + + pool.mevBundles = append(pool.mevBundles, mevBundle{ + txs: txs, + blockNumber: blockNumber, + minTimestamp: minTimestamp, + maxTimestamp: maxTimestamp, + }) + return nil +} + // Locals retrieves the accounts currently considered local by the pool. func (pool *TxPool) Locals() []common.Address { pool.mu.Lock() diff --git a/core/tx_pool_test.go b/core/tx_pool_test.go index e02096fe25ed..30fc4159a7ac 100644 --- a/core/tx_pool_test.go +++ b/core/tx_pool_test.go @@ -2541,3 +2541,13 @@ func BenchmarkInsertRemoteWithAllLocals(b *testing.B) { pool.Stop() } } + +func checkBundles(t *testing.T, pool *TxPool, block int64, timestamp uint64, expectedRes int, expectedRemaining int) { + res, _ := pool.MevBundles(big.NewInt(block), timestamp) + if len(res) != expectedRes { + t.Fatalf("expected returned bundles did not match got %d, expected %d", len(res), expectedRes) + } + if len(pool.mevBundles) != expectedRemaining { + t.Fatalf("expected remaining bundles did not match got %d, expected %d", len(pool.mevBundles), expectedRemaining) + } +} diff --git a/eth/api_backend.go b/eth/api_backend.go index f22462c7c88a..c380bb1394f6 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -234,6 +234,10 @@ func (b *EthAPIBackend) SendTx(ctx context.Context, signedTx *types.Transaction) return b.eth.txPool.AddLocal(signedTx) } +func (b *EthAPIBackend) SendBundle(ctx context.Context, txs types.Transactions, blockNumber rpc.BlockNumber, minTimestamp uint64, maxTimestamp uint64) error { + return b.eth.txPool.AddMevBundle(txs, big.NewInt(blockNumber.Int64()), minTimestamp, maxTimestamp) +} + func (b *EthAPIBackend) GetPoolTransactions() (types.Transactions, error) { pending, err := b.eth.txPool.Pending(false) if err != nil { diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index b65f98836c31..32b6aa1f6a78 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -2058,3 +2058,39 @@ func toHexSlice(b [][]byte) []string { } return r } + +// ---------------------------------------------------------------- FlashBots ---------------------------------------------------------------- + +// PrivateTxBundleAPI offers an API for accepting bundled transactions +type PrivateTxBundleAPI struct { + b Backend +} + +// NewPrivateTxBundleAPI creates a new Tx Bundle API instance. +func NewPrivateTxBundleAPI(b Backend) *PrivateTxBundleAPI { + return &PrivateTxBundleAPI{b} +} + +// SendBundle will add the signed transaction to the transaction pool. +// The sender is responsible for signing the transaction and using the correct nonce and ensuring validity +func (s *PrivateTxBundleAPI) SendBundle(ctx context.Context, encodedTxs []hexutil.Bytes, blockNumber rpc.BlockNumber, minTimestampPtr, maxTimestampPtr *uint64) error { + var txs types.Transactions + + for _, encodedTx := range encodedTxs { + tx := new(types.Transaction) + if err := rlp.DecodeBytes(encodedTx, tx); err != nil { + return err + } + txs = append(txs, tx) + } + + var minTimestamp, maxTimestamp uint64 + if minTimestampPtr != nil { + minTimestamp = *minTimestampPtr + } + if maxTimestampPtr != nil { + maxTimestamp = *maxTimestampPtr + } + + return s.b.SendBundle(ctx, txs, blockNumber, minTimestamp, maxTimestamp) +} diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index fe55ec59c82a..a31a19b70c7c 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -71,6 +71,7 @@ type Backend interface { // Transaction pool API SendTx(ctx context.Context, signedTx *types.Transaction) error + SendBundle(ctx context.Context, txs types.Transactions, blockNumber rpc.BlockNumber, minTimestamp uint64, maxTimestamp uint64) error GetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error) GetPoolTransactions() (types.Transactions, error) GetPoolTransaction(txHash common.Hash) *types.Transaction @@ -134,6 +135,11 @@ func GetAPIs(apiBackend Backend) []rpc.API { Version: "1.0", Service: NewPrivateAccountAPI(apiBackend, nonceLock), Public: false, + }, { + Namespace: "eth", + Version: "1.0", + Service: NewPrivateTxBundleAPI(apiBackend), + Public: true, }, } } diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index 927dba1897b3..4ef71b67e822 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -558,6 +558,11 @@ web3._extend({ params: 3, inputFormatter: [null, web3._extend.formatters.inputBlockNumberFormatter, null] }), + new web3._extend.Method({ + name: 'sendBundle', + call: 'eth_sendBundle', + params: 4 + }), ], properties: [ new web3._extend.Property({ diff --git a/les/api_backend.go b/les/api_backend.go index 326b540b6f85..9491d3bfed88 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -197,6 +197,9 @@ func (b *LesApiBackend) SendTx(ctx context.Context, signedTx *types.Transaction) func (b *LesApiBackend) RemoveTx(txHash common.Hash) { b.eth.txPool.RemoveTx(txHash) } +func (b *LesApiBackend) SendBundle(ctx context.Context, txs types.Transactions, blockNumber rpc.BlockNumber, minTimestamp uint64, maxTimestamp uint64) error { + return b.eth.txPool.AddMevBundle(txs, big.NewInt(blockNumber.Int64()), minTimestamp, maxTimestamp) +} func (b *LesApiBackend) GetPoolTransactions() (types.Transactions, error) { return b.eth.txPool.GetTransactions() diff --git a/light/txpool.go b/light/txpool.go index a7df4aeec388..c16be9b91660 100644 --- a/light/txpool.go +++ b/light/txpool.go @@ -550,3 +550,14 @@ func (pool *TxPool) RemoveTx(hash common.Hash) { pool.chainDb.Delete(hash[:]) pool.relay.Discard([]common.Hash{hash}) } + +// MevBundles returns a list of bundles valid for the given blockNumber/blockTimestamp +// also prunes bundles that are outdated +func (pool *TxPool) MevBundles(blockNumber *big.Int, blockTimestamp uint64) ([]types.Transactions, error) { + return nil, nil +} + +// AddMevBundle adds a mev bundle to the pool +func (pool *TxPool) AddMevBundle(txs types.Transactions, blockNumber *big.Int, minTimestamp uint64, maxTimestamp uint64) error { + return nil +} From 4d67cc408c32f4d07f2e08ede6489dc1351ebda9 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Fri, 19 Feb 2021 10:56:06 -0800 Subject: [PATCH 02/17] Add Flashbots bundles to miner --- miner/worker.go | 233 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 223 insertions(+), 10 deletions(-) diff --git a/miner/worker.go b/miner/worker.go index accf3dac9096..d210da1ca97a 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -87,6 +87,7 @@ type environment struct { uncles mapset.Set // uncle set tcount int // tx count in cycle gasPool *core.GasPool // available gas used to pack transactions + profit *big.Int header *types.Header txs []*types.Transaction @@ -99,6 +100,8 @@ type task struct { state *state.StateDB block *types.Block createdAt time.Time + + profit *big.Int } const ( @@ -551,6 +554,9 @@ func (w *worker) taskLoop() { var ( stopCh chan struct{} prev common.Hash + + prevNumber *big.Int + prevProfit *big.Int ) // interrupt aborts the in-flight sealing task. @@ -571,10 +577,18 @@ func (w *worker) taskLoop() { if sealHash == prev { continue } + + // reject new tasks which don't profit + if prevNumber != nil && prevProfit != nil && + task.block.Number().Cmp(prevNumber) == 0 && task.profit.Cmp(prevProfit) < 0 { + continue + } + prevNumber, prevProfit = task.block.Number(), task.profit + // Interrupt previous sealing operation interrupt() stopCh, prev = make(chan struct{}), sealHash - + log.Info("Proposed miner block", "blockNumber", prevNumber, "profit", prevProfit, "sealhash", sealHash) if w.skipSealHook != nil && w.skipSealHook(task) { continue } @@ -658,13 +672,12 @@ func (w *worker) resultLoop() { } } -// makeCurrent creates a new environment for the current cycle. -func (w *worker) makeCurrent(parent *types.Block, header *types.Header) error { +func (w *worker) generateEnv(parent *types.Block, header *types.Header) (*environment, error) { // Retrieve the parent state to execute on top and start a prefetcher for // the miner to speed block sealing up a bit state, err := w.chain.StateAt(parent.Root()) if err != nil { - return err + return nil, err } state.StartPrefetcher("miner") @@ -675,6 +688,7 @@ func (w *worker) makeCurrent(parent *types.Block, header *types.Header) error { family: mapset.NewSet(), uncles: mapset.NewSet(), header: header, + profit: new(big.Int), } // when 08 is processed ancestors contain 07 (quick block) for _, ancestor := range w.chain.GetBlocksFromHash(parent.Hash(), 7) { @@ -686,6 +700,16 @@ func (w *worker) makeCurrent(parent *types.Block, header *types.Header) error { } // Keep track of transactions which return errors so they can be removed env.tcount = 0 + env.gasPool = new(core.GasPool).AddGas(header.GasLimit) + return env, nil +} + +// makeCurrent creates a new environment for the current cycle. +func (w *worker) makeCurrent(parent *types.Block, header *types.Header) error { + env, err := w.generateEnv(parent, header) + if err != nil { + return err + } // Swap out the old work with the new one, terminating any leftover prefetcher // processes in the mean time and starting a new one. @@ -749,8 +773,9 @@ func (w *worker) updateSnapshot() { w.snapshotState = w.current.state.Copy() } -func (w *worker) commitTransaction(tx *types.Transaction, coinbase common.Address) ([]*types.Log, error) { +func (w *worker) commitTransaction(tx *types.Transaction, coinbase common.Address, trackProfit bool) ([]*types.Log, error) { snap := w.current.state.Snapshot() + initialBalance := w.current.state.GetBalance(w.coinbase) receipt, err := core.ApplyTransaction(w.chainConfig, w.chain, &coinbase, w.current.gasPool, w.current.state, w.current.header, tx, &w.current.header.GasUsed, *w.chain.GetVMConfig()) if err != nil { @@ -760,9 +785,131 @@ func (w *worker) commitTransaction(tx *types.Transaction, coinbase common.Addres w.current.txs = append(w.current.txs, tx) w.current.receipts = append(w.current.receipts, receipt) + // coinbase balance difference already contains gas fee + if trackProfit { + finalBalance := w.current.state.GetBalance(w.coinbase) + w.current.profit.Add(w.current.profit, new(big.Int).Sub(finalBalance, initialBalance)) + } else { + gasUsed := new(big.Int).SetUint64(receipt.GasUsed) + w.current.profit.Add(w.current.profit, gasUsed.Mul(gasUsed, tx.GasPrice())) + } + return receipt.Logs, nil } +func (w *worker) commitBundle(txs types.Transactions, coinbase common.Address, interrupt *int32) bool { + // Short circuit if current is nil + if w.current == nil { + return true + } + + gasLimit := w.current.header.GasLimit + if w.current.gasPool == nil { + w.current.gasPool = new(core.GasPool).AddGas(gasLimit) + } + + var coalescedLogs []*types.Log + + for _, tx := range txs { + // In the following three cases, we will interrupt the execution of the transaction. + // (1) new head block event arrival, the interrupt signal is 1 + // (2) worker start or restart, the interrupt signal is 1 + // (3) worker recreate the mining block with any newly arrived transactions, the interrupt signal is 2. + // For the first two cases, the semi-finished work will be discarded. + // For the third case, the semi-finished work will be submitted to the consensus engine. + if interrupt != nil && atomic.LoadInt32(interrupt) != commitInterruptNone { + // Notify resubmit loop to increase resubmitting interval due to too frequent commits. + if atomic.LoadInt32(interrupt) == commitInterruptResubmit { + ratio := float64(gasLimit-w.current.gasPool.Gas()) / float64(gasLimit) + if ratio < 0.1 { + ratio = 0.1 + } + w.resubmitAdjustCh <- &intervalAdjust{ + ratio: ratio, + inc: true, + } + } + return atomic.LoadInt32(interrupt) == commitInterruptNewHead + } + // If we don't have enough gas for any further transactions then we're done + if w.current.gasPool.Gas() < params.TxGas { + log.Trace("Not enough gas for further transactions", "have", w.current.gasPool, "want", params.TxGas) + break + } + // Error may be ignored here. The error has already been checked + // during transaction acceptance is the transaction pool. + // + // We use the eip155 signer regardless of the current hf. + from, _ := types.Sender(w.current.signer, tx) + // Check whether the tx is replay protected. If we're not in the EIP155 hf + // phase, start ignoring the sender until we do. + if tx.Protected() && !w.chainConfig.IsEIP155(w.current.header.Number) { + log.Trace("Ignoring reply protected transaction", "hash", tx.Hash(), "eip155", w.chainConfig.EIP155Block) + + return true + } + // Start executing the transaction + w.current.state.Prepare(tx.Hash(), common.Hash{}, w.current.tcount) + + logs, err := w.commitTransaction(tx, coinbase, true) + switch { + case errors.Is(err, core.ErrGasLimitReached): + // Pop the current out-of-gas transaction without shifting in the next from the account + log.Trace("Gas limit exceeded for current block", "sender", from) + return true + + case errors.Is(err, core.ErrNonceTooLow): + // New head notification data race between the transaction pool and miner, shift + log.Trace("Skipping transaction with low nonce", "sender", from, "nonce", tx.Nonce()) + return true + + case errors.Is(err, core.ErrNonceTooHigh): + // Reorg notification data race between the transaction pool and miner, skip account = + log.Trace("Skipping account with hight nonce", "sender", from, "nonce", tx.Nonce()) + return true + + case errors.Is(err, nil): + // Everything ok, collect the logs and shift in the next transaction from the same account + coalescedLogs = append(coalescedLogs, logs...) + w.current.tcount++ + continue + + case errors.Is(err, core.ErrTxTypeNotSupported): + // Pop the unsupported transaction without shifting in the next from the account + log.Trace("Skipping unsupported transaction type", "sender", from, "type", tx.Type()) + return true + + default: + // Strange error, discard the transaction and get the next in line (note, the + // nonce-too-high clause will prevent us from executing in vain). + log.Debug("Transaction failed, account skipped", "hash", tx.Hash(), "err", err) + return true + } + } + + if !w.isRunning() && len(coalescedLogs) > 0 { + // We don't push the pendingLogsEvent while we are mining. The reason is that + // when we are mining, the worker will regenerate a mining block every 3 seconds. + // In order to avoid pushing the repeated pendingLog, we disable the pending log pushing. + + // make a copy, the state caches the logs and these logs get "upgraded" from pending to mined + // logs by filling in the block hash when the block was mined by the local miner. This can + // cause a race condition if a log was "upgraded" before the PendingLogsEvent is processed. + cpy := make([]*types.Log, len(coalescedLogs)) + for i, l := range coalescedLogs { + cpy[i] = new(types.Log) + *cpy[i] = *l + } + w.pendingLogsFeed.Send(cpy) + } + // Notify resubmit loop to decrease resubmitting interval if current interval is larger + // than the user-specified one. + if interrupt != nil { + w.resubmitAdjustCh <- &intervalAdjust{inc: false} + } + return false +} + func (w *worker) commitTransactions(txs *types.TransactionsByPriceAndNonce, coinbase common.Address, interrupt *int32) bool { // Short circuit if current is nil if w.current == nil { @@ -823,7 +970,7 @@ func (w *worker) commitTransactions(txs *types.TransactionsByPriceAndNonce, coin // Start executing the transaction w.current.state.Prepare(tx.Hash(), w.current.tcount) - logs, err := w.commitTransaction(tx, coinbase) + logs, err := w.commitTransaction(tx, coinbase, false) switch { case errors.Is(err, core.ErrGasLimitReached): // Pop the current out-of-gas transaction without shifting in the next from the account @@ -886,7 +1033,6 @@ func (w *worker) commitTransactions(txs *types.TransactionsByPriceAndNonce, coin func (w *worker) commitNewWork(interrupt *int32, noempty bool, timestamp int64) { w.mu.RLock() defer w.mu.RUnlock() - tstart := time.Now() parent := w.chain.CurrentBlock() @@ -984,10 +1130,10 @@ func (w *worker) commitNewWork(interrupt *int32, noempty bool, timestamp int64) log.Error("Failed to fetch pending transactions", "err", err) return } - // Short circuit if there is no available pending transactions. + // Short circuit if there is no available pending transactions or bundles. // But if we disable empty precommit already, ignore it. Since // empty block is necessary to keep the liveness of the network. - if len(pending) == 0 && atomic.LoadUint32(&w.noempty) == 0 { + if len(pending) == 0 && atomic.LoadUint32(&w.noempty) == 0 && len(w.eth.TxPool().AllMevBundles()) == 0 { w.updateSnapshot() return } @@ -999,6 +1145,16 @@ func (w *worker) commitNewWork(interrupt *int32, noempty bool, timestamp int64) localTxs[account] = txs } } + bundles, err := w.eth.TxPool().MevBundles(header.Number, header.Time) + if err != nil { + log.Error("Failed to fetch pending transactions", "err", err) + return + } + maxBundle, bundlePrice, ethToCoinbase, gasUsed := w.findMostProfitableBundle(bundles, w.coinbase, parent, header) + log.Info("Flashbots bundle", "ethToCoinbase", ethToCoinbase, "gasUsed", gasUsed, "bundlePrice", bundlePrice, "bundleLength", len(maxBundle)) + if w.commitBundle(maxBundle, w.coinbase, interrupt) { + return + } if len(localTxs) > 0 { txs := types.NewTransactionsByPriceAndNonce(w.current.signer, localTxs, header.BaseFee) if w.commitTransactions(txs, w.coinbase, interrupt) { @@ -1029,7 +1185,7 @@ func (w *worker) commit(uncles []*types.Header, interval func(), update bool, st interval() } select { - case w.taskCh <- &task{receipts: receipts, state: s, block: block, createdAt: time.Now()}: + case w.taskCh <- &task{receipts: receipts, state: s, block: block, createdAt: time.Now(), profit: w.current.profit}: w.unconfirmed.Shift(block.NumberU64() - 1) log.Info("Commit new mining work", "number", block.Number(), "sealhash", w.engine.SealHash(block.Header()), "uncles", len(uncles), "txs", w.current.tcount, @@ -1046,6 +1202,63 @@ func (w *worker) commit(uncles []*types.Header, interval func(), update bool, st return nil } +func (w *worker) findMostProfitableBundle(bundles []types.Transactions, coinbase common.Address, parent *types.Block, header *types.Header) (types.Transactions, *big.Int, *big.Int, uint64) { + maxBundlePrice := new(big.Int) + maxTotalEth := new(big.Int) + var maxTotalGasUsed uint64 + maxBundle := types.Transactions{} + for _, bundle := range bundles { + if len(bundle) == 0 { + continue + } + totalEth, totalGasUsed, err := w.computeBundleGas(bundle, parent, header) + + if err != nil { + log.Debug("Error computing gas for a bundle", "error", err) + continue + } + + mevGasPrice := new(big.Int).Div(totalEth, new(big.Int).SetUint64(totalGasUsed)) + if mevGasPrice.Cmp(maxBundlePrice) > 0 { + maxBundle = bundle + maxBundlePrice = mevGasPrice + maxTotalEth = totalEth + maxTotalGasUsed = totalGasUsed + } + } + + return maxBundle, maxBundlePrice, maxTotalEth, maxTotalGasUsed +} + +// Compute the adjusted gas price for a whole bundle +// Done by calculating all gas spent, adding transfers to the coinbase, and then dividing by gas used +func (w *worker) computeBundleGas(bundle types.Transactions, parent *types.Block, header *types.Header) (*big.Int, uint64, error) { + env, err := w.generateEnv(parent, header) + defer env.state.StopPrefetcher() + if err != nil { + return nil, 0, err + } + + var totalGasUsed uint64 = 0 + var tempGasUsed uint64 + + coinbaseBalanceBefore := env.state.GetBalance(w.coinbase) + + for _, tx := range bundle { + receipt, err := core.ApplyTransaction(w.chainConfig, w.chain, &w.coinbase, env.gasPool, env.state, env.header, tx, &tempGasUsed, *w.chain.GetVMConfig()) + if err != nil { + return nil, 0, err + } + totalGasUsed += receipt.GasUsed + } + coinbaseBalanceAfter := env.state.GetBalance(w.coinbase) + coinbaseDiff := new(big.Int).Sub(coinbaseBalanceAfter, coinbaseBalanceBefore) + totalEth := new(big.Int) + totalEth.Add(totalEth, coinbaseDiff) + + return totalEth, totalGasUsed, nil +} + // copyReceipts makes a deep copy of the given receipts. func copyReceipts(receipts []*types.Receipt) []*types.Receipt { result := make([]*types.Receipt, len(receipts)) From 7e671fa4669834c72b7c6ee626c04b42d2af21d1 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Fri, 19 Feb 2021 10:56:53 -0800 Subject: [PATCH 03/17] Add Flashbots profit switching to miner --- miner/miner.go | 10 +++--- miner/multi_worker.go | 80 +++++++++++++++++++++++++++++++++++++++++++ miner/worker.go | 80 ++++++++++++++++++++++++++++++------------- miner/worker_test.go | 5 ++- 4 files changed, 146 insertions(+), 29 deletions(-) create mode 100644 miner/multi_worker.go diff --git a/miner/miner.go b/miner/miner.go index a4a01b9f4ff7..fc747c9ec1f3 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -56,7 +56,7 @@ type Config struct { // Miner creates blocks and searches for proof-of-work values. type Miner struct { mux *event.TypeMux - worker *worker + worker *multiWorker coinbase common.Address eth Backend engine consensus.Engine @@ -73,7 +73,7 @@ func New(eth Backend, config *Config, chainConfig *params.ChainConfig, mux *even exitCh: make(chan struct{}), startCh: make(chan common.Address), stopCh: make(chan struct{}), - worker: newWorker(config, chainConfig, engine, eth, mux, isLocalBlock, true), + worker: newMultiWorker(config, chainConfig, engine, eth, mux, isLocalBlock, true), } go miner.update() @@ -182,7 +182,7 @@ func (miner *Miner) SetRecommitInterval(interval time.Duration) { // Pending returns the currently pending block and associated state. func (miner *Miner) Pending() (*types.Block, *state.StateDB) { - return miner.worker.pending() + return miner.worker.regularWorker.pending() } // PendingBlock returns the currently pending block. @@ -191,7 +191,7 @@ func (miner *Miner) Pending() (*types.Block, *state.StateDB) { // simultaneously, please use Pending(), as the pending state can // change between multiple method calls func (miner *Miner) PendingBlock() *types.Block { - return miner.worker.pendingBlock() + return miner.worker.regularWorker.pendingBlock() } // PendingBlockAndReceipts returns the currently pending block and corresponding receipts. @@ -230,5 +230,5 @@ func (miner *Miner) DisablePreseal() { // SubscribePendingLogs starts delivering logs from pending transactions // to the given channel. func (miner *Miner) SubscribePendingLogs(ch chan<- []*types.Log) event.Subscription { - return miner.worker.pendingLogsFeed.Subscribe(ch) + return miner.worker.regularWorker.pendingLogsFeed.Subscribe(ch) } diff --git a/miner/multi_worker.go b/miner/multi_worker.go new file mode 100644 index 000000000000..ea2b2e4f6299 --- /dev/null +++ b/miner/multi_worker.go @@ -0,0 +1,80 @@ +package miner + +import ( + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/params" +) + +type multiWorker struct { + regularWorker *worker + flashbotsWorker *worker +} + +func (w *multiWorker) stop() { + w.regularWorker.stop() + w.flashbotsWorker.stop() +} + +func (w *multiWorker) start() { + w.regularWorker.start() + w.flashbotsWorker.start() +} + +func (w *multiWorker) close() { + w.regularWorker.close() + w.flashbotsWorker.close() +} + +func (w *multiWorker) isRunning() bool { + return w.regularWorker.isRunning() || w.flashbotsWorker.isRunning() +} + +func (w *multiWorker) setExtra(extra []byte) { + w.regularWorker.setExtra(extra) + w.flashbotsWorker.setExtra(extra) +} + +func (w *multiWorker) setRecommitInterval(interval time.Duration) { + w.regularWorker.setRecommitInterval(interval) + w.flashbotsWorker.setRecommitInterval(interval) +} + +func (w *multiWorker) setEtherbase(addr common.Address) { + w.regularWorker.setEtherbase(addr) + w.flashbotsWorker.setEtherbase(addr) +} + +func (w *multiWorker) enablePreseal() { + w.regularWorker.enablePreseal() + w.flashbotsWorker.enablePreseal() +} + +func (w *multiWorker) disablePreseal() { + w.regularWorker.disablePreseal() + w.flashbotsWorker.disablePreseal() +} + +func newMultiWorker(config *Config, chainConfig *params.ChainConfig, engine consensus.Engine, eth Backend, mux *event.TypeMux, isLocalBlock func(*types.Block) bool, init bool) *multiWorker { + queue := make(chan *task) + + return &multiWorker{ + regularWorker: newWorker(config, chainConfig, engine, eth, mux, isLocalBlock, init, &flashbotsData{ + isFlashbots: false, + queue: queue, + }), + flashbotsWorker: newWorker(config, chainConfig, engine, eth, mux, isLocalBlock, init, &flashbotsData{ + isFlashbots: true, + queue: queue, + }), + } +} + +type flashbotsData struct { + isFlashbots bool + queue chan *task +} diff --git a/miner/worker.go b/miner/worker.go index d210da1ca97a..784a1d7ae443 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -101,7 +101,8 @@ type task struct { block *types.Block createdAt time.Time - profit *big.Int + profit *big.Int + isFlashbots bool } const ( @@ -184,6 +185,8 @@ type worker struct { // External functions isLocalBlock func(block *types.Block) bool // Function used to determine whether the specified block is mined by local miner. + flashbots *flashbotsData + // Test hooks newTaskHook func(*task) // Method to call upon receiving a new sealing task. skipSealHook func(*task) bool // Method to decide whether skipping the sealing. @@ -191,7 +194,30 @@ type worker struct { resubmitHook func(time.Duration, time.Duration) // Method to call upon updating resubmitting interval. } -func newWorker(config *Config, chainConfig *params.ChainConfig, engine consensus.Engine, eth Backend, mux *event.TypeMux, isLocalBlock func(*types.Block) bool, init bool) *worker { +func newWorker(config *Config, chainConfig *params.ChainConfig, engine consensus.Engine, eth Backend, mux *event.TypeMux, isLocalBlock func(*types.Block) bool, init bool, flashbots *flashbotsData) *worker { + exitCh := make(chan struct{}) + taskCh := make(chan *task) + if flashbots.isFlashbots { + // publish to the flashbots queue + taskCh = flashbots.queue + } else { + // read from the flashbots queue + go func() { + for { + select { + case flashbotsTask := <-flashbots.queue: + select { + case taskCh <- flashbotsTask: + case <-exitCh: + return + } + case <-exitCh: + return + } + } + }() + } + worker := &worker{ config: config, chainConfig: chainConfig, @@ -208,12 +234,13 @@ func newWorker(config *Config, chainConfig *params.ChainConfig, engine consensus chainHeadCh: make(chan core.ChainHeadEvent, chainHeadChanSize), chainSideCh: make(chan core.ChainSideEvent, chainSideChanSize), newWorkCh: make(chan *newWorkReq), - taskCh: make(chan *task), + taskCh: taskCh, resultCh: make(chan *types.Block, resultQueueSize), - exitCh: make(chan struct{}), + exitCh: exitCh, startCh: make(chan struct{}, 1), resubmitIntervalCh: make(chan time.Duration), resubmitAdjustCh: make(chan *intervalAdjust, resubmitAdjustChanSize), + flashbots: flashbots, } // Subscribe NewTxsEvent for tx pool worker.txsSub = eth.TxPool().SubscribeNewTxsEvent(worker.txsCh) @@ -230,8 +257,11 @@ func newWorker(config *Config, chainConfig *params.ChainConfig, engine consensus go worker.mainLoop() go worker.newWorkLoop(recommit) - go worker.resultLoop() - go worker.taskLoop() + if !flashbots.isFlashbots { + // only mine if not flashbots + go worker.resultLoop() + go worker.taskLoop() + } // Submit first work to initialize pending state. if init { @@ -588,7 +618,7 @@ func (w *worker) taskLoop() { // Interrupt previous sealing operation interrupt() stopCh, prev = make(chan struct{}), sealHash - log.Info("Proposed miner block", "blockNumber", prevNumber, "profit", prevProfit, "sealhash", sealHash) + log.Info("Proposed miner block", "blockNumber", prevNumber, "profit", prevProfit, "isFlashbots", task.isFlashbots, "sealhash", sealHash) if w.skipSealHook != nil && w.skipSealHook(task) { continue } @@ -673,13 +703,10 @@ func (w *worker) resultLoop() { } func (w *worker) generateEnv(parent *types.Block, header *types.Header) (*environment, error) { - // Retrieve the parent state to execute on top and start a prefetcher for - // the miner to speed block sealing up a bit state, err := w.chain.StateAt(parent.Root()) if err != nil { return nil, err } - state.StartPrefetcher("miner") env := &environment{ signer: types.MakeSigner(w.chainConfig, header.Number), @@ -707,6 +734,7 @@ func (w *worker) generateEnv(parent *types.Block, header *types.Header) (*enviro // makeCurrent creates a new environment for the current cycle. func (w *worker) makeCurrent(parent *types.Block, header *types.Header) error { env, err := w.generateEnv(parent, header) + env.state.StartPrefetcher("miner") if err != nil { return err } @@ -1133,7 +1161,11 @@ func (w *worker) commitNewWork(interrupt *int32, noempty bool, timestamp int64) // Short circuit if there is no available pending transactions or bundles. // But if we disable empty precommit already, ignore it. Since // empty block is necessary to keep the liveness of the network. - if len(pending) == 0 && atomic.LoadUint32(&w.noempty) == 0 && len(w.eth.TxPool().AllMevBundles()) == 0 { + noBundles := true + if w.flashbots.isFlashbots && len(w.eth.TxPool().AllMevBundles()) > 0 { + noBundles = false + } + if len(pending) == 0 && atomic.LoadUint32(&w.noempty) == 0 && noBundles { w.updateSnapshot() return } @@ -1145,15 +1177,17 @@ func (w *worker) commitNewWork(interrupt *int32, noempty bool, timestamp int64) localTxs[account] = txs } } - bundles, err := w.eth.TxPool().MevBundles(header.Number, header.Time) - if err != nil { - log.Error("Failed to fetch pending transactions", "err", err) - return - } - maxBundle, bundlePrice, ethToCoinbase, gasUsed := w.findMostProfitableBundle(bundles, w.coinbase, parent, header) - log.Info("Flashbots bundle", "ethToCoinbase", ethToCoinbase, "gasUsed", gasUsed, "bundlePrice", bundlePrice, "bundleLength", len(maxBundle)) - if w.commitBundle(maxBundle, w.coinbase, interrupt) { - return + if w.flashbots.isFlashbots { + bundles, err := w.eth.TxPool().MevBundles(header.Number, header.Time) + if err != nil { + log.Error("Failed to fetch pending transactions", "err", err) + return + } + maxBundle, bundlePrice, ethToCoinbase, gasUsed := w.findMostProfitableBundle(bundles, w.coinbase, parent, header) + log.Info("Flashbots bundle", "ethToCoinbase", ethToCoinbase, "gasUsed", gasUsed, "bundlePrice", bundlePrice, "bundleLength", len(maxBundle)) + if w.commitBundle(maxBundle, w.coinbase, interrupt) { + return + } } if len(localTxs) > 0 { txs := types.NewTransactionsByPriceAndNonce(w.current.signer, localTxs, header.BaseFee) @@ -1185,12 +1219,13 @@ func (w *worker) commit(uncles []*types.Header, interval func(), update bool, st interval() } select { - case w.taskCh <- &task{receipts: receipts, state: s, block: block, createdAt: time.Now(), profit: w.current.profit}: + case w.taskCh <- &task{receipts: receipts, state: s, block: block, createdAt: time.Now(), profit: w.current.profit, isFlashbots: w.flashbots.isFlashbots}: w.unconfirmed.Shift(block.NumberU64() - 1) log.Info("Commit new mining work", "number", block.Number(), "sealhash", w.engine.SealHash(block.Header()), "uncles", len(uncles), "txs", w.current.tcount, "gas", block.GasUsed(), "fees", totalFees(block, receipts), - "elapsed", common.PrettyDuration(time.Since(start))) + "elapsed", common.PrettyDuration(time.Since(start)), + "isFlashbots", w.flashbots.isFlashbots) case <-w.exitCh: log.Info("Worker has exited") @@ -1234,7 +1269,6 @@ func (w *worker) findMostProfitableBundle(bundles []types.Transactions, coinbase // Done by calculating all gas spent, adding transfers to the coinbase, and then dividing by gas used func (w *worker) computeBundleGas(bundle types.Transactions, parent *types.Block, header *types.Header) (*big.Int, uint64, error) { env, err := w.generateEnv(parent, header) - defer env.state.StopPrefetcher() if err != nil { return nil, 0, err } diff --git a/miner/worker_test.go b/miner/worker_test.go index 2bb6c9407bbe..72a38334bd59 100644 --- a/miner/worker_test.go +++ b/miner/worker_test.go @@ -196,7 +196,10 @@ func (b *testWorkerBackend) newRandomTx(creation bool) *types.Transaction { func newTestWorker(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine, db ethdb.Database, blocks int) (*worker, *testWorkerBackend) { backend := newTestWorkerBackend(t, chainConfig, engine, db, blocks) backend.txPool.AddLocals(pendingTxs) - w := newWorker(testConfig, chainConfig, engine, backend, new(event.TypeMux), nil, false) + w := newWorker(testConfig, chainConfig, engine, backend, new(event.TypeMux), nil, false, &flashbotsData{ + isFlashbots: false, + queue: nil, + }) w.setEtherbase(testBankAddress) return w, backend } From f401f0cb8cb868d5a5e9d9c06a088947728131fd Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Fri, 19 Feb 2021 11:04:55 -0800 Subject: [PATCH 04/17] Add infra/CI and update README --- .github/workflows/go.yml | 64 ++ README.md | 400 ++++-------- README.original.md | 359 +++++++++++ infra/Dockerfile.node | 23 + infra/Dockerfile.updater | 23 + infra/mev-geth-nodes-arm64.yaml | 979 +++++++++++++++++++++++++++++ infra/mev-geth-nodes-x86-64.yaml | 972 ++++++++++++++++++++++++++++ infra/mev-geth-updater-arm64.yaml | 749 ++++++++++++++++++++++ infra/mev-geth-updater-x86-64.yaml | 737 ++++++++++++++++++++++ infra/start-mev-geth-node.sh | 96 +++ infra/start-mev-geth-updater.sh | 181 ++++++ 11 files changed, 4302 insertions(+), 281 deletions(-) create mode 100644 .github/workflows/go.yml create mode 100644 README.original.md create mode 100644 infra/Dockerfile.node create mode 100644 infra/Dockerfile.updater create mode 100644 infra/mev-geth-nodes-arm64.yaml create mode 100644 infra/mev-geth-nodes-x86-64.yaml create mode 100644 infra/mev-geth-updater-arm64.yaml create mode 100644 infra/mev-geth-updater-x86-64.yaml create mode 100755 infra/start-mev-geth-node.sh create mode 100755 infra/start-mev-geth-updater.sh diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml new file mode 100644 index 000000000000..3fc1f2ff8c68 --- /dev/null +++ b/.github/workflows/go.yml @@ -0,0 +1,64 @@ +name: Go + +on: + push: + pull_request: + branches: [ master ] + +jobs: + + build: + name: Build + runs-on: ubuntu-latest + steps: + + - name: Set up Go 1.x + uses: actions/setup-go@v2 + with: + go-version: ^1.13 + id: go + + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + + - name: Test + run: go test ./core ./miner/... ./internal/ethapi/... ./les/... + + - name: Build + run: make geth + + e2e: + name: End to End + runs-on: ubuntu-latest + steps: + + - name: Set up Go 1.x + uses: actions/setup-go@v2 + with: + go-version: ^1.13 + id: go + + - name: Use Node.js 12.x + uses: actions/setup-node@v1 + with: + node-version: 12.x + + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + + - name: Build + run: make geth + + - name: Check out the e2e code repo + uses: actions/checkout@v2 + with: + repository: flashbots/mev-geth-demo + path: e2e + + - run: cd e2e && yarn install + - run: | + cd e2e + GETH=`pwd`/../build/bin/geth ./run.sh & + sleep 15 + yarn run demo-simple + yarn run demo-contract diff --git a/README.md b/README.md index cb55f565cd42..324de818f5be 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,12 @@ -## Go Ethereum +# MEV-geth -Official Golang implementation of the Ethereum protocol. +This is a fork of go-ethereum, [the original README is here](README.original.md). -[![API Reference]( -https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/68747470733a2f2f676f646f632e6f72672f6769746875622e636f6d2f676f6c616e672f6764646f3f7374617475732e737667 -)](https://pkg.go.dev/github.com/ethereum/go-ethereum?tab=doc) -[![Go Report Card](https://goreportcard.com/badge/github.com/ethereum/go-ethereum)](https://goreportcard.com/report/github.com/ethereum/go-ethereum) -[![Travis](https://travis-ci.com/ethereum/go-ethereum.svg?branch=master)](https://travis-ci.com/ethereum/go-ethereum) -[![Discord](https://img.shields.io/badge/discord-join%20chat-blue.svg)](https://discord.gg/nthXNEv) +Flashbots is a research and development organization formed to mitigate the negative externalities and existential risks posed by miner-extractable value (MEV) to smart-contract blockchains. We propose a permissionless, transparent, and fair ecosystem for MEV extraction that reinforce the Ethereum ideals. -Automated builds are available for stable releases and the unstable master branch. Binary -archives are published at https://geth.ethereum.org/downloads/. +## Quick start + +<<<<<<< HEAD ## Building the source @@ -20,59 +16,65 @@ Building `geth` requires both a Go (version 1.14 or later) and a C compiler. You them using your favourite package manager. Once the dependencies are installed, run ```shell -make geth -``` -or, to build the full suite of utilities: +git clone https://github.com/flashbots/mev-geth +cd mev-geth + +> > > > > > > dfdcfc666 (Add infra/CI and update README) +> > > > > > > make geth -```shell -make all ``` -## Executables +See [here](https://geth.ethereum.org/docs/install-and-build/installing-geth#build-go-ethereum-from-source-code) for further info on building MEV-geth from source. -The go-ethereum project comes with several wrappers/executables found in the `cmd` -directory. +## MEV-Geth: a proof of concept -| Command | Description | -| :-----------: | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **`geth`** | Our main Ethereum CLI client. It is the entry point into the Ethereum network (main-, test- or private net), capable of running as a full node (default), archive node (retaining all historical state) or a light node (retrieving data live). It can be used by other processes as a gateway into the Ethereum network via JSON RPC endpoints exposed on top of HTTP, WebSocket and/or IPC transports. `geth --help` and the [CLI page](https://geth.ethereum.org/docs/interface/command-line-options) for command line options. | -| `clef` | Stand-alone signing tool, which can be used as a backend signer for `geth`. | -| `devp2p` | Utilities to interact with nodes on the networking layer, without running a full blockchain. | -| `abigen` | Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages. It operates on plain [Ethereum contract ABIs](https://docs.soliditylang.org/en/develop/abi-spec.html) with expanded functionality if the contract bytecode is also available. However, it also accepts Solidity source files, making development much more streamlined. Please see our [Native DApps](https://geth.ethereum.org/docs/dapp/native-bindings) page for details. | -| `bootnode` | Stripped down version of our Ethereum client implementation that only takes part in the network node discovery protocol, but does not run any of the higher level application protocols. It can be used as a lightweight bootstrap node to aid in finding peers in private networks. | -| `evm` | Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode. Its purpose is to allow isolated, fine-grained debugging of EVM opcodes (e.g. `evm --code 60ff60ff --debug run`). | -| `rlpdump` | Developer utility tool to convert binary RLP ([Recursive Length Prefix](https://eth.wiki/en/fundamentals/rlp)) dumps (data encoding used by the Ethereum protocol both network as well as consensus wise) to user-friendlier hierarchical representation (e.g. `rlpdump --hex CE0183FFFFFFC4C304050583616263`). | -| `puppeth` | a CLI wizard that aids in creating a new Ethereum network. | +We have designed and implemented a proof of concept for permissionless MEV extraction called MEV-Geth. It is a sealed-bid block space auction mechanism for communicating transaction order preference. While our proof of concept has incomplete trust guarantees, we believe it's a significant improvement over the status quo. The adoption of MEV-Geth should relieve a lot of the network and chain congestion caused by frontrunning and backrunning bots. -## Running `geth` +| Guarantee | PGA | Dark-txPool | MEV-Geth | +| -------------------- | :-: | :---------: | :------: | +| Permissionless | ✅ | ❌ | ✅ | +| Efficient | ❌ | ❌ | ✅ | +| Pre-trade privacy | ❌ | ✅ | ✅ | +| Failed trade privacy | ❌ | ❌ | ✅ | +| Complete privacy | ❌ | ❌ | ❌ | +| Finality | ❌ | ❌ | ❌ | -Going through all the possible command line flags is out of scope here (please consult our -[CLI Wiki page](https://geth.ethereum.org/docs/interface/command-line-options)), -but we've enumerated a few common parameter combos to get you up to speed quickly -on how you can run your own `geth` instance. +### Why MEV-Geth? -### Full node on the main Ethereum network +We believe that without the adoption of neutral, public, open-source infrastructure for permissionless MEV extraction, MEV risks becoming an insiders' game. We commit as an organization to releasing reference implementations for participation in fair, ethical, and politically neutral MEV extraction. By doing so, we hope to prevent the properties of Ethereum from being eroded by trust-based dark pools or proprietary channels which are key points of security weakness. We thus release MEV-Geth with the dual goal of creating an ecosystem for MEV extraction that preserves Ethereum properties, as well as starting conversations with the community around our research and development roadmap. -By far the most common scenario is people wanting to simply interact with the Ethereum -network: create accounts; transfer funds; deploy and interact with contracts. For this -particular use-case the user doesn't care about years-old historical data, so we can -fast-sync quickly to the current state of the network. To do so: +### Design goals -```shell -$ geth console -``` +- **Permissionless** + A permissionless design implies there are no trusted intermediary which can censor transactions. +- **Efficient** + An efficient design implies MEV extraction is performed without causing unnecessary network or chain congestion. +- **Pre-trade privacy** + Pre-trade privacy implies transactions only become publicly known after they have been included in a block. Note, this type of privacy does not exclude privileged actors such as transaction aggregators / gateways / miners. +- **Failed trade privacy** + Failed trade privacy implies loosing bids are never included in a block, thus never exposed to the public. Failed trade privacy is tightly coupled to extraction efficiency. +- **Complete privacy** + Complete privacy implies there are no privileged actors such as transaction aggregators / gateways / miners who can observe incoming transactions. +- **Finality** + Finality implies it is infeasible for MEV extraction to be reversed once included in a block. This would protect against time-bandit chain re-org attacks. + +The MEV-Geth proof of concept relies on the fact that searchers can withhold bids from certain miners in order to disincentivize bad behavior like stealing a profitable strategy. We expect a complete privacy design to necessitate some sort of private computation solution like SGX, ZKP, or MPC to withhold the transaction content from miners until it is mined in a block. One of the core objective of the Flashbots organization is to incentivize and produce research in this direction. +The MEV-Geth proof of concept does not provide any finality guarantees. We expect the solution to this problem to require post-trade execution privacy through private chain state or strong economic infeasibility. The design of a system with strong finality is the second core objective of the MEV-Geth research effort. + +<<<<<<< HEAD This command will: - * Start `geth` in fast sync mode (default, can be changed with the `--syncmode` flag), - causing it to download more data in exchange for avoiding processing the entire history - of the Ethereum network, which is very CPU intensive. - * Start up `geth`'s built-in interactive [JavaScript console](https://geth.ethereum.org/docs/interface/javascript-console), - (via the trailing `console` subcommand) through which you can interact using [`web3` methods](https://web3js.readthedocs.io/en/) - (note: the `web3` version bundled within `geth` is very old, and not up to date with official docs), - as well as `geth`'s own [management APIs](https://geth.ethereum.org/docs/rpc/server). - This tool is optional and if you leave it out you can always attach to an already running - `geth` instance with `geth attach`. + +- Start `geth` in fast sync mode (default, can be changed with the `--syncmode` flag), + causing it to download more data in exchange for avoiding processing the entire history + of the Ethereum network, which is very CPU intensive. +- Start up `geth`'s built-in interactive [JavaScript console](https://geth.ethereum.org/docs/interface/javascript-console), + (via the trailing `console` subcommand) through which you can interact using [`web3` methods](https://web3js.readthedocs.io/en/) + (note: the `web3` version bundled within `geth` is very old, and not up to date with official docs), + as well as `geth`'s own [management APIs](https://geth.ethereum.org/docs/rpc/server). + This tool is optional and if you leave it out you can always attach to an already running + `geth` instance with `geth attach`. ### A Full node on the Görli test network @@ -86,277 +88,113 @@ the main network, but with play-Ether only. $ geth --goerli console ``` -The `console` subcommand has the exact same meaning as above and they are equally -useful on the testnet too. Please, see above for their explanations if you've skipped here. +======= -Specifying the `--goerli` flag, however, will reconfigure your `geth` instance a bit: +### How it works - * Instead of connecting the main Ethereum network, the client will connect to the Görli - test network, which uses different P2P bootnodes, different network IDs and genesis - states. - * Instead of using the default data directory (`~/.ethereum` on Linux for example), `geth` - will nest itself one level deeper into a `goerli` subfolder (`~/.ethereum/goerli` on - Linux). Note, on OSX and Linux this also means that attaching to a running testnet node - requires the use of a custom endpoint since `geth attach` will try to attach to a - production node endpoint by default, e.g., - `geth attach /goerli/geth.ipc`. Windows users are not affected by - this. +> > > > > > > dfdcfc666 (Add infra/CI and update README) -*Note: Although there are some internal protective measures to prevent transactions from -crossing over between the main network and test network, you should make sure to always -use separate accounts for play-money and real-money. Unless you manually move -accounts, `geth` will by default correctly separate the two networks and will not make any -accounts available between them.* +MEV-Geth introduces the concepts of "searchers", "transaction bundles", and "block template" to Ethereum. Effectively, MEV-Geth provides a way for miners to delegate the task of finding and ordering transactions to third parties called "searchers". These searchers compete with each other to find the most profitable ordering and bid for its inclusion in the next block using a standardized template called a "transaction bundle". These bundles are evaluated in a sealed-bid auction hosted by miners to produce a "block template" which holds the [information about transaction order required to begin mining](https://ethereum.stackexchange.com/questions/268/ethereum-block-architecture). -### Full node on the Rinkeby test network +![](https://hackmd.io/_uploads/B1fWz7rcD.png) -Go Ethereum also supports connecting to the older proof-of-authority based test network -called [*Rinkeby*](https://www.rinkeby.io) which is operated by members of the community. +The MEV-Geth proof of concept is compatible with any regular Ethereum client. The Flashbots core devs are maintaining [a reference implementation](https://github.com/flashbots/mev-geth) for the go-ethereum client. -```shell -$ geth --rinkeby console -``` +### Differences between MEV-Geth and [_vanilla_ geth](https://github.com/ethereum/go-ethereum) -### Full node on the Ropsten test network +The entire patch can be broken down into four modules: -In addition to Görli and Rinkeby, Geth also supports the ancient Ropsten testnet. The -Ropsten test network is based on the Ethash proof-of-work consensus algorithm. As such, -it has certain extra overhead and is more susceptible to reorganization attacks due to the -network's low difficulty/security. +1. bundle worker and `eth_sendBundle` rpc (commits [8104d5d7b0a54bd98b3a08479a1fde685eb53c29](https://github.com/flashbots/mev-geth/commit/8104d5d7b0a54bd98b3a08479a1fde685eb53c29) and [c2b5b4029b2b748a6f1a9d5668f12096f096563d](https://github.com/flashbots/mev-geth/commit/c2b5b4029b2b748a6f1a9d5668f12096f096563d)) +2. profit switcher (commit [aa5840d22f4882f91ecba0eb20ef35a702b134d5](https://github.com/flashbots/mev-geth/commit/aa5840d22f4882f91ecba0eb20ef35a702b134d5)) +3. `eth_callBundle` simulation rpc (commits [9199d2e13d484df7a634fad12343ed2b46d5d4c3](https://github.com/flashbots/mev-geth/commit/9199d2e13d484df7a634fad12343ed2b46d5d4c3) and [a99dfc198817dd171128cc22439c81896e876619](https://github.com/flashbots/mev-geth/commit/a99dfc198817dd171128cc22439c81896e876619)) +4. Documentation (this file) and CI/infrastructure configuration (commit [035109807944f7a446467aa27ca8ec98d109a465](https://github.com/flashbots/mev-geth/commit/035109807944f7a446467aa27ca8ec98d109a465)) -```shell -$ geth --ropsten console -``` +The entire changeset can be viewed inspecting the [diff](https://github.com/ethereum/go-ethereum/compare/master...flashbots:master). -*Note: Older Geth configurations store the Ropsten database in the `testnet` subdirectory.* +In summary: -### Configuration +- Geth’s txpool is modified to also contain a `mevBundles` field, which stores a list of MEV bundles. Each MEV bundle is an array of transactions, along with a min/max timestamp for their inclusion. +- A new `eth_sendBundle` API is exposed which allows adding an MEV Bundle to the txpool. During the Flashbots Alpha, this is only called by MEV-relay. + - The transactions submitted to the bundle are “eth_sendRawTransaction-style” RLP encoded signed transactions along with the min/max block of inclusion + - This API is a no-op when run in light mode +- Geth’s miner is modified as follows: + - While in the event loop, before adding all the pending txpool “normal” transactions to the block, it: + - Finds the most profitable bundle + - It picks the most profitable bundle by returning the one with the highest average gas price per unit of gas + - computeBundleGas: Returns average gas price (\sum{gasprice_i\*gasused_i + (coinbase_after - coinbase_before)) / \sum{gasused_i}) + - Commits the bundle (remember: Bundle transactions are not ordered by nonce or gas price). For each transaction in the bundle, it: + - `Prepare`’s it against the state + - CommitsTransaction with trackProfit = true + w.current.profit += coinbase_after_tx - coinbase_before_tx + w.current.profit += gas \* gas_price + - If a block is found where the w.current.profit is more than the previous profit, it switches mining to that block. +- A new `eth_callBundle` API is exposed that enables simulation of transaction bundles. +- Documentation and CI/infrastructure files are added. -As an alternative to passing the numerous flags to the `geth` binary, you can also pass a -configuration file via: +### MEV-Geth for miners -```shell -$ geth --config /path/to/your_config.toml -``` - -To get an idea how the file should look like you can use the `dumpconfig` subcommand to -export your existing configuration: - -```shell -$ geth --your-favourite-flags dumpconfig -``` +Miners can start mining MEV blocks by running MEV-Geth, or by implementing their own fork that matches the specification. -*Note: This works only with `geth` v1.6.0 and above.* +While only the bundle worker and `eth_sendBundle` module (1) is necessary to mine flashbots blocks, we recommend also running the profit switcher module (2) to guarantee mining rewards are maximized. The `eth_callBundle` simulation rpc module (3) is not needed for the alpha. The suggested configuration is implemented in the `master` branch of this repository, which also includes the documentation module (4). -#### Docker quick start +We issue and maintain [releases](https://github.com/flashbots/mev-geth/releases) for the recommended configuration for the current and immediately prior versions of geth. -One of the quickest ways to get Ethereum up and running on your machine is by using -Docker: +In order to see the diff of the recommended patch, run: -```shell -docker run -d --name ethereum-node -v /Users/alice/ethereum:/root \ - -p 8545:8545 -p 30303:30303 \ - ethereum/client-go ``` - -This will start `geth` in fast-sync mode with a DB memory allowance of 1GB just as the -above command does. It will also create a persistent volume in your home directory for -saving your blockchain as well as map the default ports. There is also an `alpine` tag -available for a slim version of the image. - -Do not forget `--http.addr 0.0.0.0`, if you want to access RPC from other containers -and/or hosts. By default, `geth` binds to the local interface and RPC endpoints is not -accessible from the outside. - -### Programmatically interfacing `geth` nodes - -As a developer, sooner rather than later you'll want to start interacting with `geth` and the -Ethereum network via your own programs and not manually through the console. To aid -this, `geth` has built-in support for a JSON-RPC based APIs ([standard APIs](https://eth.wiki/json-rpc/API) -and [`geth` specific APIs](https://geth.ethereum.org/docs/rpc/server)). -These can be exposed via HTTP, WebSockets and IPC (UNIX sockets on UNIX based -platforms, and named pipes on Windows). - -The IPC interface is enabled by default and exposes all the APIs supported by `geth`, -whereas the HTTP and WS interfaces need to manually be enabled and only expose a -subset of APIs due to security reasons. These can be turned on/off and configured as -you'd expect. - -HTTP based JSON-RPC API options: - - * `--http` Enable the HTTP-RPC server - * `--http.addr` HTTP-RPC server listening interface (default: `localhost`) - * `--http.port` HTTP-RPC server listening port (default: `8545`) - * `--http.api` API's offered over the HTTP-RPC interface (default: `eth,net,web3`) - * `--http.corsdomain` Comma separated list of domains from which to accept cross origin requests (browser enforced) - * `--ws` Enable the WS-RPC server - * `--ws.addr` WS-RPC server listening interface (default: `localhost`) - * `--ws.port` WS-RPC server listening port (default: `8546`) - * `--ws.api` API's offered over the WS-RPC interface (default: `eth,net,web3`) - * `--ws.origins` Origins from which to accept websockets requests - * `--ipcdisable` Disable the IPC-RPC server - * `--ipcapi` API's offered over the IPC-RPC interface (default: `admin,debug,eth,miner,net,personal,shh,txpool,web3`) - * `--ipcpath` Filename for IPC socket/pipe within the datadir (explicit paths escape it) - -You'll need to use your own programming environments' capabilities (libraries, tools, etc) to -connect via HTTP, WS or IPC to a `geth` node configured with the above flags and you'll -need to speak [JSON-RPC](https://www.jsonrpc.org/specification) on all transports. You -can reuse the same connection for multiple requests! - -**Note: Please understand the security implications of opening up an HTTP/WS based -transport before doing so! Hackers on the internet are actively trying to subvert -Ethereum nodes with exposed APIs! Further, all browser tabs can access locally -running web servers, so malicious web pages could try to subvert locally available -APIs!** - -### Operating a private network - -Maintaining your own private network is more involved as a lot of configurations taken for -granted in the official networks need to be manually set up. - -#### Defining the private genesis state - -First, you'll need to create the genesis state of your networks, which all nodes need to be -aware of and agree upon. This consists of a small JSON file (e.g. call it `genesis.json`): - -```json -{ - "config": { - "chainId": , - "homesteadBlock": 0, - "eip150Block": 0, - "eip155Block": 0, - "eip158Block": 0, - "byzantiumBlock": 0, - "constantinopleBlock": 0, - "petersburgBlock": 0, - "istanbulBlock": 0, - "berlinBlock": 0 - }, - "alloc": {}, - "coinbase": "0x0000000000000000000000000000000000000000", - "difficulty": "0x20000", - "extraData": "", - "gasLimit": "0x2fefd8", - "nonce": "0x0000000000000042", - "mixhash": "0x0000000000000000000000000000000000000000000000000000000000000000", - "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", - "timestamp": "0x00" -} + git diff master~4..master~1 ``` -The above fields should be fine for most purposes, although we'd recommend changing -the `nonce` to some random value so you prevent unknown remote nodes from being able -to connect to you. If you'd like to pre-fund some accounts for easier testing, create -the accounts and populate the `alloc` field with their addresses. +Alternatively, the `master-barebones` branch includes only modules (1) and (4), leaving the profit switching logic to miners. While this usage is discouraged, it entails a much smaller change in the code. -```json -"alloc": { - "0x0000000000000000000000000000000000000001": { - "balance": "111111111" - }, - "0x0000000000000000000000000000000000000002": { - "balance": "222222222" - } -} -``` - -With the genesis state defined in the above JSON file, you'll need to initialize **every** -`geth` node with it prior to starting it up to ensure all blockchain parameters are correctly -set: - -```shell -$ geth init path/to/genesis.json -``` - -#### Creating the rendezvous point - -With all nodes that you want to run initialized to the desired genesis state, you'll need to -start a bootstrap node that others can use to find each other in your network and/or over -the internet. The clean way is to configure and run a dedicated bootnode: - -```shell -$ bootnode --genkey=boot.key -$ bootnode --nodekey=boot.key -``` +At this stage, we recommend only receiving bundles via a relay, to prevent abuse via denial-of-service attacks. We have [implemented](https://github.com/flashbots/mev-relay) and currently run such relay. This relay performs basic rate limiting and miner profitability checks, but does otherwise not interfere with submitted bundles in any way, and is open for everybody to participate. We invite you to try the [Flashbots Alpha](https://github.com/flashbots/pm#flashbots-alpha) and start receiving MEV revenue by following these steps: -With the bootnode online, it will display an [`enode` URL](https://eth.wiki/en/fundamentals/enode-url-format) -that other nodes can use to connect to it and exchange peer information. Make sure to -replace the displayed IP address information (most probably `[::]`) with your externally -accessible IP to get the actual `enode` URL. +1. Fill out this [form](https://forms.gle/78JS52d22dwrgabi6) to indicate your interest in participating in the Alpha and be added to the MEV-Relay miner whitelist. +2. You will receive an onboarding email from Flashbots to help [set up](https://github.com/flashbots/mev-geth/blob/master/README.md#quick-start) your MEV-Geth node and protect it with a [reverse proxy](https://github.com/flashbots/mev-relay-js/blob/master/miner/proxy.js) to open the `eth_sendBundle` RPC. +3. Respond to Flashbots' email with your MEV-Geth node endpoint to be added to the Flashbots hosted [MEV-relay](https://github.com/flashbots/mev-relay-js) gateway. MEV-Relay is needed during the alpha to aggregate bundle requests from all users, prevent spam and DOS attacks on participating miner(s)/mining pool(s), and collect system health metrics. +4. After receiving a confirmation email that your MEV-Geth node's endpoint has been added to the relay, you will immediately start receiving Flashbots transaction bundles with associated MEV revenue paid to you. -*Note: You could also use a full-fledged `geth` node as a bootnode, but it's the less -recommended way.* +### MEV-Geth for searchers -#### Starting up your member nodes +You do _not_ need to run MEV-Geth as a searcher, but, instead, to monitor the Ethereum state and transaction pool for MEV opportunities and produce transaction bundles that extract that MEV. Anyone can become a searcher. In fact, the bundles produced by searchers don't need to extract MEV at all, but we expect the most valuable bundles will. -With the bootnode operational and externally reachable (you can try -`telnet ` to ensure it's indeed reachable), start every subsequent `geth` -node pointed to the bootnode for peer discovery via the `--bootnodes` flag. It will -probably also be desirable to keep the data directory of your private network separated, so -do also specify a custom `--datadir` flag. +An MEV-Geth bundle is a standard message template composed of an array of valid ethereum transactions, a blockheight, and an optional timestamp range over which the bundle is valid. -```shell -$ geth --datadir=path/to/custom/data/folder --bootnodes= -``` - -*Note: Since your network will be completely cut off from the main and test networks, you'll -also need to configure a miner to process transactions and create new blocks for you.* - -#### Running a private miner - -Mining on the public Ethereum network is a complex task as it's only feasible using GPUs, -requiring an OpenCL or CUDA enabled `ethminer` instance. For information on such a -setup, please consult the [EtherMining subreddit](https://www.reddit.com/r/EtherMining/) -and the [ethminer](https://github.com/ethereum-mining/ethminer) repository. - -In a private network setting, however a single CPU miner instance is more than enough for -practical purposes as it can produce a stable stream of blocks at the correct intervals -without needing heavy resources (consider running on a single thread, no need for multiple -ones either). To start a `geth` instance for mining, run it with all your usual flags, extended -by: - -```shell -$ geth --mine --miner.threads=1 --miner.etherbase=0x0000000000000000000000000000000000000000 +```json +{ + "signedTransactions": ["..."], // RLP encoded signed transaction array + "blocknumber": "0x386526", // hex string + "minTimestamp": 12345, // optional uint64 + "maxTimestamp": 12345 // optional uint64 +} ``` -Which will start mining blocks and transactions on a single CPU thread, crediting all -proceedings to the account specified by `--miner.etherbase`. You can further tune the mining -by changing the default gas limit blocks converge to (`--miner.targetgaslimit`) and the price -transactions are accepted at (`--miner.gasprice`). +The `signedTransactions` can be any valid ethereum transactions. Care must be taken to place transaction nonces in correct order. -## Contribution +The `blocknumber` defines the block height at which the bundle is to be included. A bundle will only be evaluated for the provided blockheight and immediately evicted if not selected. -Thank you for considering to help out with the source code! We welcome contributions -from anyone on the internet, and are grateful for even the smallest of fixes! +The `minTimestamp` and `maxTimestamp` are optional conditions to further restrict bundle validity within a time range. +<<<<<<< HEAD If you'd like to contribute to go-ethereum, please fork, fix, commit and send a pull request for the maintainers to review and merge into the main code base. If you wish to submit more complex changes though, please check up with the core devs first on [our Discord Server](https://discord.gg/invite/nthXNEv) to ensure those changes are in line with the general philosophy of the project and/or get some early feedback which can make both your efforts much lighter as well as our review and merge procedures quick and simple. +======= +MEV-Geth miners select the most profitable bundle per unit of gas used and place it at the beginning of the list of transactions of the block template at a given blockheight. Miners determine the value of a bundle based on the following equation. _Note, the change in block.coinbase balance represents a direct transfer of ETH through a smart contract._ + +> > > > > > > fcac1062f (Add infra/CI and update README) -Please make sure your contributions adhere to our coding guidelines: + - * Code must adhere to the official Go [formatting](https://golang.org/doc/effective_go.html#formatting) - guidelines (i.e. uses [gofmt](https://golang.org/cmd/gofmt/)). - * Code must be documented adhering to the official Go [commentary](https://golang.org/doc/effective_go.html#commentary) - guidelines. - * Pull requests need to be based on and opened against the `master` branch. - * Commit messages should be prefixed with the package(s) they modify. - * E.g. "eth, rpc: make trace configs optional" +To submit a bundle, the searcher sends the bundle directly to the miner using the rpc method `eth_sendBundle`. Since MEV-Geth requires direct communication between searchers and miners, a searcher can configure the list of miners where they want to send their bundle. -Please see the [Developers' Guide](https://geth.ethereum.org/docs/developers/devguide) -for more details on configuring your environment, managing project dependencies, and -testing procedures. +### Feature requests and bug reports -## License +If you are a user of MEV-Geth and have suggestions on how to make integration with your current setup easier, or would like to submit a bug report, we encourage you to open an issue in this repository with the `enhancement` or `bug` labels respectively. If you need help getting started, please ask in the dedicated [#⛏️miners](https://discord.gg/rcgADN9qFX) channel in our Discord. -The go-ethereum library (i.e. all code outside of the `cmd` directory) is licensed under the -[GNU Lesser General Public License v3.0](https://www.gnu.org/licenses/lgpl-3.0.en.html), -also included in our repository in the `COPYING.LESSER` file. +### Moving beyond proof of concept -The go-ethereum binaries (i.e. all code inside of the `cmd` directory) is licensed under the -[GNU General Public License v3.0](https://www.gnu.org/licenses/gpl-3.0.en.html), also -included in our repository in the `COPYING` file. +We provide the MEV-Geth proof of concept as a first milestone on the path to mitigating the negative externalities caused by MEV. We hope to discuss with the community the merits of adopting MEV-Geth in its current form. Our preliminary research indicates it could free at least 2.5% of the current chain congestion by eliminating the use of frontrunning and backrunning and provide uplift of up to 18% on miner rewards from Ethereum. That being said, we believe a sustainable solution to MEV existential risks requires complete privacy and finality, which the proof of concept does not address. We hope to engage community feedback throughout the development of this complete version of MEV-Geth. diff --git a/README.original.md b/README.original.md new file mode 100644 index 000000000000..ddb885dfdc36 --- /dev/null +++ b/README.original.md @@ -0,0 +1,359 @@ +## Go Ethereum + +Official Golang implementation of the Ethereum protocol. + +[![API Reference]( +https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/68747470733a2f2f676f646f632e6f72672f6769746875622e636f6d2f676f6c616e672f6764646f3f7374617475732e737667 +)](https://pkg.go.dev/github.com/ethereum/go-ethereum?tab=doc) +[![Go Report Card](https://goreportcard.com/badge/github.com/ethereum/go-ethereum)](https://goreportcard.com/report/github.com/ethereum/go-ethereum) +[![Travis](https://travis-ci.org/ethereum/go-ethereum.svg?branch=master)](https://travis-ci.org/ethereum/go-ethereum) +[![Discord](https://img.shields.io/badge/discord-join%20chat-blue.svg)](https://discord.gg/nthXNEv) + +Automated builds are available for stable releases and the unstable master branch. Binary +archives are published at https://geth.ethereum.org/downloads/. + +## Building the source + +For prerequisites and detailed build instructions please read the [Installation Instructions](https://github.com/ethereum/go-ethereum/wiki/Building-Ethereum) on the wiki. + +Building `geth` requires both a Go (version 1.13 or later) and a C compiler. You can install +them using your favourite package manager. Once the dependencies are installed, run + +```shell +make geth +``` + +or, to build the full suite of utilities: + +```shell +make all +``` + +## Executables + +The go-ethereum project comes with several wrappers/executables found in the `cmd` +directory. + +| Command | Description | +| :-----------: | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **`geth`** | Our main Ethereum CLI client. It is the entry point into the Ethereum network (main-, test- or private net), capable of running as a full node (default), archive node (retaining all historical state) or a light node (retrieving data live). It can be used by other processes as a gateway into the Ethereum network via JSON RPC endpoints exposed on top of HTTP, WebSocket and/or IPC transports. `geth --help` and the [CLI Wiki page](https://github.com/ethereum/go-ethereum/wiki/Command-Line-Options) for command line options. | +| `abigen` | Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages. It operates on plain [Ethereum contract ABIs](https://github.com/ethereum/wiki/wiki/Ethereum-Contract-ABI) with expanded functionality if the contract bytecode is also available. However, it also accepts Solidity source files, making development much more streamlined. Please see our [Native DApps](https://github.com/ethereum/go-ethereum/wiki/Native-DApps:-Go-bindings-to-Ethereum-contracts) wiki page for details. | +| `bootnode` | Stripped down version of our Ethereum client implementation that only takes part in the network node discovery protocol, but does not run any of the higher level application protocols. It can be used as a lightweight bootstrap node to aid in finding peers in private networks. | +| `evm` | Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode. Its purpose is to allow isolated, fine-grained debugging of EVM opcodes (e.g. `evm --code 60ff60ff --debug run`). | +| `gethrpctest` | Developer utility tool to support our [ethereum/rpc-test](https://github.com/ethereum/rpc-tests) test suite which validates baseline conformity to the [Ethereum JSON RPC](https://github.com/ethereum/wiki/wiki/JSON-RPC) specs. Please see the [test suite's readme](https://github.com/ethereum/rpc-tests/blob/master/README.md) for details. | +| `rlpdump` | Developer utility tool to convert binary RLP ([Recursive Length Prefix](https://github.com/ethereum/wiki/wiki/RLP)) dumps (data encoding used by the Ethereum protocol both network as well as consensus wise) to user-friendlier hierarchical representation (e.g. `rlpdump --hex CE0183FFFFFFC4C304050583616263`). | +| `puppeth` | a CLI wizard that aids in creating a new Ethereum network. | + +## Running `geth` + +Going through all the possible command line flags is out of scope here (please consult our +[CLI Wiki page](https://github.com/ethereum/go-ethereum/wiki/Command-Line-Options)), +but we've enumerated a few common parameter combos to get you up to speed quickly +on how you can run your own `geth` instance. + +### Full node on the main Ethereum network + +By far the most common scenario is people wanting to simply interact with the Ethereum +network: create accounts; transfer funds; deploy and interact with contracts. For this +particular use-case the user doesn't care about years-old historical data, so we can +fast-sync quickly to the current state of the network. To do so: + +```shell +$ geth console +``` + +This command will: + * Start `geth` in fast sync mode (default, can be changed with the `--syncmode` flag), + causing it to download more data in exchange for avoiding processing the entire history + of the Ethereum network, which is very CPU intensive. + * Start up `geth`'s built-in interactive [JavaScript console](https://github.com/ethereum/go-ethereum/wiki/JavaScript-Console), + (via the trailing `console` subcommand) through which you can invoke all official [`web3` methods](https://github.com/ethereum/wiki/wiki/JavaScript-API) + as well as `geth`'s own [management APIs](https://github.com/ethereum/go-ethereum/wiki/Management-APIs). + This tool is optional and if you leave it out you can always attach to an already running + `geth` instance with `geth attach`. + +### A Full node on the Görli test network + +Transitioning towards developers, if you'd like to play around with creating Ethereum +contracts, you almost certainly would like to do that without any real money involved until +you get the hang of the entire system. In other words, instead of attaching to the main +network, you want to join the **test** network with your node, which is fully equivalent to +the main network, but with play-Ether only. + +```shell +$ geth --goerli console +``` + +The `console` subcommand has the exact same meaning as above and they are equally +useful on the testnet too. Please, see above for their explanations if you've skipped here. + +Specifying the `--goerli` flag, however, will reconfigure your `geth` instance a bit: + + * Instead of connecting the main Ethereum network, the client will connect to the Görli + test network, which uses different P2P bootnodes, different network IDs and genesis + states. + * Instead of using the default data directory (`~/.ethereum` on Linux for example), `geth` + will nest itself one level deeper into a `goerli` subfolder (`~/.ethereum/goerli` on + Linux). Note, on OSX and Linux this also means that attaching to a running testnet node + requires the use of a custom endpoint since `geth attach` will try to attach to a + production node endpoint by default, e.g., + `geth attach /goerli/geth.ipc`. Windows users are not affected by + this. + +*Note: Although there are some internal protective measures to prevent transactions from +crossing over between the main network and test network, you should make sure to always +use separate accounts for play-money and real-money. Unless you manually move +accounts, `geth` will by default correctly separate the two networks and will not make any +accounts available between them.* + +### Full node on the Rinkeby test network + +Go Ethereum also supports connecting to the older proof-of-authority based test network +called [*Rinkeby*](https://www.rinkeby.io) which is operated by members of the community. + +```shell +$ geth --rinkeby console +``` + +### Full node on the Ropsten test network + +In addition to Görli and Rinkeby, Geth also supports the ancient Ropsten testnet. The +Ropsten test network is based on the Ethash proof-of-work consensus algorithm. As such, +it has certain extra overhead and is more susceptible to reorganization attacks due to the +network's low difficulty/security. + +```shell +$ geth --ropsten console +``` + +*Note: Older Geth configurations store the Ropsten database in the `testnet` subdirectory.* + +### Configuration + +As an alternative to passing the numerous flags to the `geth` binary, you can also pass a +configuration file via: + +```shell +$ geth --config /path/to/your_config.toml +``` + +To get an idea how the file should look like you can use the `dumpconfig` subcommand to +export your existing configuration: + +```shell +$ geth --your-favourite-flags dumpconfig +``` + +*Note: This works only with `geth` v1.6.0 and above.* + +#### Docker quick start + +One of the quickest ways to get Ethereum up and running on your machine is by using +Docker: + +```shell +docker run -d --name ethereum-node -v /Users/alice/ethereum:/root \ + -p 8545:8545 -p 30303:30303 \ + ethereum/client-go +``` + +This will start `geth` in fast-sync mode with a DB memory allowance of 1GB just as the +above command does. It will also create a persistent volume in your home directory for +saving your blockchain as well as map the default ports. There is also an `alpine` tag +available for a slim version of the image. + +Do not forget `--http.addr 0.0.0.0`, if you want to access RPC from other containers +and/or hosts. By default, `geth` binds to the local interface and RPC endpoints is not +accessible from the outside. + +### Programmatically interfacing `geth` nodes + +As a developer, sooner rather than later you'll want to start interacting with `geth` and the +Ethereum network via your own programs and not manually through the console. To aid +this, `geth` has built-in support for a JSON-RPC based APIs ([standard APIs](https://github.com/ethereum/wiki/wiki/JSON-RPC) +and [`geth` specific APIs](https://github.com/ethereum/go-ethereum/wiki/Management-APIs)). +These can be exposed via HTTP, WebSockets and IPC (UNIX sockets on UNIX based +platforms, and named pipes on Windows). + +The IPC interface is enabled by default and exposes all the APIs supported by `geth`, +whereas the HTTP and WS interfaces need to manually be enabled and only expose a +subset of APIs due to security reasons. These can be turned on/off and configured as +you'd expect. + +HTTP based JSON-RPC API options: + + * `--http` Enable the HTTP-RPC server + * `--http.addr` HTTP-RPC server listening interface (default: `localhost`) + * `--http.port` HTTP-RPC server listening port (default: `8545`) + * `--http.api` API's offered over the HTTP-RPC interface (default: `eth,net,web3`) + * `--http.corsdomain` Comma separated list of domains from which to accept cross origin requests (browser enforced) + * `--ws` Enable the WS-RPC server + * `--ws.addr` WS-RPC server listening interface (default: `localhost`) + * `--ws.port` WS-RPC server listening port (default: `8546`) + * `--ws.api` API's offered over the WS-RPC interface (default: `eth,net,web3`) + * `--ws.origins` Origins from which to accept websockets requests + * `--ipcdisable` Disable the IPC-RPC server + * `--ipcapi` API's offered over the IPC-RPC interface (default: `admin,debug,eth,miner,net,personal,shh,txpool,web3`) + * `--ipcpath` Filename for IPC socket/pipe within the datadir (explicit paths escape it) + +You'll need to use your own programming environments' capabilities (libraries, tools, etc) to +connect via HTTP, WS or IPC to a `geth` node configured with the above flags and you'll +need to speak [JSON-RPC](https://www.jsonrpc.org/specification) on all transports. You +can reuse the same connection for multiple requests! + +**Note: Please understand the security implications of opening up an HTTP/WS based +transport before doing so! Hackers on the internet are actively trying to subvert +Ethereum nodes with exposed APIs! Further, all browser tabs can access locally +running web servers, so malicious web pages could try to subvert locally available +APIs!** + +### Operating a private network + +Maintaining your own private network is more involved as a lot of configurations taken for +granted in the official networks need to be manually set up. + +#### Defining the private genesis state + +First, you'll need to create the genesis state of your networks, which all nodes need to be +aware of and agree upon. This consists of a small JSON file (e.g. call it `genesis.json`): + +```json +{ + "config": { + "chainId": , + "homesteadBlock": 0, + "eip150Block": 0, + "eip155Block": 0, + "eip158Block": 0, + "byzantiumBlock": 0, + "constantinopleBlock": 0, + "petersburgBlock": 0, + "istanbulBlock": 0 + }, + "alloc": {}, + "coinbase": "0x0000000000000000000000000000000000000000", + "difficulty": "0x20000", + "extraData": "", + "gasLimit": "0x2fefd8", + "nonce": "0x0000000000000042", + "mixhash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "timestamp": "0x00" +} +``` + +The above fields should be fine for most purposes, although we'd recommend changing +the `nonce` to some random value so you prevent unknown remote nodes from being able +to connect to you. If you'd like to pre-fund some accounts for easier testing, create +the accounts and populate the `alloc` field with their addresses. + +```json +"alloc": { + "0x0000000000000000000000000000000000000001": { + "balance": "111111111" + }, + "0x0000000000000000000000000000000000000002": { + "balance": "222222222" + } +} +``` + +With the genesis state defined in the above JSON file, you'll need to initialize **every** +`geth` node with it prior to starting it up to ensure all blockchain parameters are correctly +set: + +```shell +$ geth init path/to/genesis.json +``` + +#### Creating the rendezvous point + +With all nodes that you want to run initialized to the desired genesis state, you'll need to +start a bootstrap node that others can use to find each other in your network and/or over +the internet. The clean way is to configure and run a dedicated bootnode: + +```shell +$ bootnode --genkey=boot.key +$ bootnode --nodekey=boot.key +``` + +With the bootnode online, it will display an [`enode` URL](https://github.com/ethereum/wiki/wiki/enode-url-format) +that other nodes can use to connect to it and exchange peer information. Make sure to +replace the displayed IP address information (most probably `[::]`) with your externally +accessible IP to get the actual `enode` URL. + +*Note: You could also use a full-fledged `geth` node as a bootnode, but it's the less +recommended way.* + +#### Starting up your member nodes + +With the bootnode operational and externally reachable (you can try +`telnet ` to ensure it's indeed reachable), start every subsequent `geth` +node pointed to the bootnode for peer discovery via the `--bootnodes` flag. It will +probably also be desirable to keep the data directory of your private network separated, so +do also specify a custom `--datadir` flag. + +```shell +$ geth --datadir=path/to/custom/data/folder --bootnodes= +``` + +*Note: Since your network will be completely cut off from the main and test networks, you'll +also need to configure a miner to process transactions and create new blocks for you.* + +#### Running a private miner + +Mining on the public Ethereum network is a complex task as it's only feasible using GPUs, +requiring an OpenCL or CUDA enabled `ethminer` instance. For information on such a +setup, please consult the [EtherMining subreddit](https://www.reddit.com/r/EtherMining/) +and the [ethminer](https://github.com/ethereum-mining/ethminer) repository. + +In a private network setting, however a single CPU miner instance is more than enough for +practical purposes as it can produce a stable stream of blocks at the correct intervals +without needing heavy resources (consider running on a single thread, no need for multiple +ones either). To start a `geth` instance for mining, run it with all your usual flags, extended +by: + +```shell +$ geth --mine --miner.threads=1 --etherbase=0x0000000000000000000000000000000000000000 +``` + +Which will start mining blocks and transactions on a single CPU thread, crediting all +proceedings to the account specified by `--etherbase`. You can further tune the mining +by changing the default gas limit blocks converge to (`--targetgaslimit`) and the price +transactions are accepted at (`--gasprice`). + +## Contribution + +Thank you for considering to help out with the source code! We welcome contributions +from anyone on the internet, and are grateful for even the smallest of fixes! + +If you'd like to contribute to go-ethereum, please fork, fix, commit and send a pull request +for the maintainers to review and merge into the main code base. If you wish to submit +more complex changes though, please check up with the core devs first on [our gitter channel](https://gitter.im/ethereum/go-ethereum) +to ensure those changes are in line with the general philosophy of the project and/or get +some early feedback which can make both your efforts much lighter as well as our review +and merge procedures quick and simple. + +Please make sure your contributions adhere to our coding guidelines: + + * Code must adhere to the official Go [formatting](https://golang.org/doc/effective_go.html#formatting) + guidelines (i.e. uses [gofmt](https://golang.org/cmd/gofmt/)). + * Code must be documented adhering to the official Go [commentary](https://golang.org/doc/effective_go.html#commentary) + guidelines. + * Pull requests need to be based on and opened against the `master` branch. + * Commit messages should be prefixed with the package(s) they modify. + * E.g. "eth, rpc: make trace configs optional" + +Please see the [Developers' Guide](https://github.com/ethereum/go-ethereum/wiki/Developers'-Guide) +for more details on configuring your environment, managing project dependencies, and +testing procedures. + +## License + +The go-ethereum library (i.e. all code outside of the `cmd` directory) is licensed under the +[GNU Lesser General Public License v3.0](https://www.gnu.org/licenses/lgpl-3.0.en.html), +also included in our repository in the `COPYING.LESSER` file. + +The go-ethereum binaries (i.e. all code inside of the `cmd` directory) is licensed under the +[GNU General Public License v3.0](https://www.gnu.org/licenses/gpl-3.0.en.html), also +included in our repository in the `COPYING` file. diff --git a/infra/Dockerfile.node b/infra/Dockerfile.node new file mode 100644 index 000000000000..db8e99ac937e --- /dev/null +++ b/infra/Dockerfile.node @@ -0,0 +1,23 @@ +# Build Geth in a stock Go builder container +FROM golang:1.15-alpine as builder + +RUN apk add --no-cache make gcc musl-dev linux-headers git + +ADD . /go-ethereum +RUN cd /go-ethereum && make geth + +# Pull Geth into a second stage deploy alpine container +FROM alpine:latest + +ENV PYTHONUNBUFFERED=1 +RUN apk add --update --no-cache groff less python3 curl jq ca-certificates && ln -sf python3 /usr/bin/python +RUN python3 -m ensurepip +RUN pip3 install --no-cache --upgrade pip setuptools awscli + +COPY --from=builder /go-ethereum/build/bin/geth /usr/local/bin/ + +COPY ./infra/start-mev-geth-node.sh /root/start-mev-geth-node.sh +RUN chmod 755 /root/start-mev-geth-node.sh + +EXPOSE 8545 8546 30303 30303/udp +ENTRYPOINT ["/root/start-mev-geth-node.sh"] diff --git a/infra/Dockerfile.updater b/infra/Dockerfile.updater new file mode 100644 index 000000000000..d3099d19ce1a --- /dev/null +++ b/infra/Dockerfile.updater @@ -0,0 +1,23 @@ +# Build Geth in a stock Go builder container +FROM golang:1.15-alpine as builder + +RUN apk add --no-cache make gcc musl-dev linux-headers git + +ADD . /go-ethereum +RUN cd /go-ethereum && make geth + +# Pull Geth into a second stage deploy alpine container +FROM alpine:latest + +ENV PYTHONUNBUFFERED=1 +RUN apk add --update --no-cache groff less python3 curl jq ca-certificates && ln -sf python3 /usr/bin/python +RUN python3 -m ensurepip +RUN pip3 install --no-cache --upgrade pip setuptools awscli + +COPY --from=builder /go-ethereum/build/bin/geth /usr/local/bin/ + +COPY ./infra/start-mev-geth-updater.sh /root/start-mev-geth-updater.sh +RUN chmod 755 /root/start-mev-geth-updater.sh + +EXPOSE 8545 8546 30303 30303/udp +ENTRYPOINT ["/root/start-mev-geth-updater.sh"] diff --git a/infra/mev-geth-nodes-arm64.yaml b/infra/mev-geth-nodes-arm64.yaml new file mode 100644 index 000000000000..af76b6aada82 --- /dev/null +++ b/infra/mev-geth-nodes-arm64.yaml @@ -0,0 +1,979 @@ +--- +AWSTemplateFormatVersion: 2010-09-09 + +Description: > + This template creates an automated continuous deployment pipeline to Amazon Elastic Container Service (ECS) + Created by Luke Youngblood, luke@blockscale.net + +Parameters: + +# GitHub Parameters + + GitHubUser: + Type: String + Default: lyoungblood + Description: Your team or username on GitHub. + + NodeGitHubRepo: + Type: String + Default: mev-geth + Description: The repo name of the node service. + + NodeGitHubBranch: + Type: String + Default: master + Description: The branch of the node repo to continuously deploy. + + GitHubToken: + Type: String + NoEcho: true + Description: > + Token for the team or user specified above. (https://github.com/settings/tokens) + +# VPC Parameters + + VPC: + Type: AWS::EC2::VPC::Id + + Subnets: + Type: List + + VpcCIDR: + Type: String + Default: 172.31.0.0/16 + +# ECS Parameters + + InstanceType: + Type: String + Default: m6gd.large + + MemoryLimit: + Type: Number + Default: 6144 + + KeyPair: + Type: AWS::EC2::KeyPair::KeyName + + SpotPrice: + Type: Number + Default: 0.0904 + + ClusterSize: + Type: Number + Default: 5 + + Bandwidth: + Type: Number + Default: 2048 + + BandwidthCeiling: + Type: Number + Default: 4096 + + NodeDesiredCount: + Type: Number + Default: 0 + + NodeTaskName: + Type: String + Default: mev-geth-node + + ECSAMI: + Type: AWS::SSM::Parameter::Value + Default: /aws/service/ecs/optimized-ami/amazon-linux-2/arm64/recommended/image_id + +# SNS Parameters + + SNSSubscriptionEndpoint: + Type: String + Default: https://events.pagerduty.com/integration/44cbdb66f22b4f3caf5dd15741c7eb17/enqueue + + SNSSubscriptionProtocol: + Type: String + Default: HTTPS + +# CloudWatch Alarm Parameters + + CPUAlarmThreshold: + Type: Number + Default: 80 + + MemoryAlarmThreshold: + Type: Number + Default: 80 + +# Mev-Geth Parameters + + Network: + Type: String + Default: mainnet + AllowedValues: + - mainnet + - goerli + + SyncMode: + Type: String + Default: fast + AllowedValues: + - full + - fast + - light + + Connections: + Type: Number + Default: 50 + + RpcPort: + Type: Number + Default: 8545 + + WsPort: + Type: Number + Default: 8546 + + NetPort: + Type: Number + Default: 30303 + +Metadata: + + AWS::CloudFormation::Interface: + ParameterLabels: + GitHubUser: + default: "User" + NodeGitHubRepo: + default: "Node Repo" + NodeGitHubBranch: + default: "Node Branch" + GitHubToken: + default: "Personal Access Token" + VPC: + default: "Choose which VPC the autoscaling group should be deployed to" + Subnets: + default: "Choose which subnets the autoscaling group should be deployed to" + VpcCIDR: + default: "VPC CIDR Block" + InstanceType: + default: "Which instance type should we use to build the ECS cluster?" + MemoryLimit: + default: "How much memory should be reserved for each task. Set to greater than 50% of instance memory capacity." + KeyPair: + default: "Which keypair should be used to allow SSH to the nodes?" + ClusterSize: + default: "How many ECS hosts do you want to initially deploy?" + SpotPrice: + default: "The maximum spot price to pay for instances - this should normally be set to the on demand price." + Bandwidth: + default: "How much bandwidth, in kb/sec., should be allocated to Ethereum peers (upload) per EC2 instance" + BandwidthCeiling: + default: "How much bandwidth, in kb/sec., should be allocated to Ethereum peers as a ceiling (max. upload)" + NodeDesiredCount: + default: "How many ECS Tasks do you want to initially execute?" + NodeTaskName: + default: "The name of the node ECS Task" + ECSAMI: + default: "The ECS AMI ID populated from SSM." + Network: + default: "The Ethereum network you will be connecting to" + SyncMode: + default: "The synchronization mode that Mev-Geth should use (full, fast, or light)" + Connections: + default: "The number of desired connections on the Mev-Geth node" + RpcPort: + default: "The RPC port used for communication with the local Mev-Geth node" + WsPort: + default: "The Websockets port used for communication with the local Mev-Geth node" + NetPort: + default: "The TCP port used for connectivity to other Ethereum peer nodes" + ParameterGroups: + - Label: + default: GitHub Configuration + Parameters: + - NodeGitHubRepo + - NodeGitHubBranch + - GitHubUser + - GitHubToken + - Label: + default: VPC Configuration + Parameters: + - VPC + - Subnets + - VpcCIDR + - Label: + default: ECS Configuration + Parameters: + - InstanceType + - MemoryLimit + - KeyPair + - SpotPrice + - ClusterSize + - Bandwidth + - BandwidthCeiling + - NodeDesiredCount + - NodeTaskName + - ECSAMI + - Label: + default: Mev-Geth Configuration + Parameters: + - Network + - SyncMode + - Connections + - RpcPort + - WsPort + - NetPort + - Label: + default: PagerDuty Endpoint Configuration + Parameters: + - SNSSubscriptionEndpoint + - SNSSubscriptionProtocol + - Label: + default: CloudWatch Alarms Configuration + Parameters: + - CPUAlarmThreshold + - MemoryAlarmThreshold + +# Mappings + +Mappings: + + RegionMap: + us-east-2: + mainnet: mev-geth-updater-fast-chainbucket-17p2xhnhcydlz + goerli: mev-geth-updater-fast-goerli-chainbucket-j6dujg8apbna + #us-west-2: + # mainnet: + # goerli: + +Resources: + +# ECS Resources + + Cluster: + Type: AWS::ECS::Cluster + Properties: + ClusterName: !Ref AWS::StackName + + SecurityGroup: + Type: "AWS::EC2::SecurityGroup" + Properties: + GroupDescription: !Sub ${AWS::StackName}-sg + VpcId: !Ref VPC + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 22 + ToPort: 22 + CidrIp: !Ref VpcCIDR + - IpProtocol: tcp + FromPort: !Ref RpcPort + ToPort: !Ref RpcPort + CidrIp: !Ref VpcCIDR + - IpProtocol: tcp + FromPort: !Ref WsPort + ToPort: !Ref WsPort + CidrIp: !Ref VpcCIDR + - IpProtocol: tcp + FromPort: !Ref NetPort + ToPort: !Ref NetPort + CidrIp: 0.0.0.0/0 + - IpProtocol: tcp + FromPort: !Ref NetPort + ToPort: !Ref NetPort + CidrIpv6: ::/0 + - IpProtocol: udp + FromPort: !Ref NetPort + ToPort: !Ref NetPort + CidrIp: 0.0.0.0/0 + - IpProtocol: udp + FromPort: !Ref NetPort + ToPort: !Ref NetPort + CidrIpv6: ::/0 + + ECSAutoScalingGroup: + Type: AWS::AutoScaling::AutoScalingGroup + Properties: + VPCZoneIdentifier: !Ref Subnets + LaunchConfigurationName: !Ref ECSLaunchConfiguration + MinSize: !Ref ClusterSize + MaxSize: !Ref ClusterSize + DesiredCapacity: !Ref ClusterSize + Tags: + - Key: Name + Value: !Sub ${AWS::StackName} ECS host + PropagateAtLaunch: true + CreationPolicy: + ResourceSignal: + Timeout: PT15M + UpdatePolicy: + AutoScalingRollingUpdate: + MinInstancesInService: 2 + MaxBatchSize: 1 + PauseTime: PT15M + SuspendProcesses: + - HealthCheck + - ReplaceUnhealthy + - AZRebalance + - AlarmNotification + - ScheduledActions + WaitOnResourceSignals: true + + ECSLaunchConfiguration: + Type: AWS::AutoScaling::LaunchConfiguration + Properties: + ImageId: !Ref ECSAMI + InstanceType: !Ref InstanceType + KeyName: !Ref KeyPair + AssociatePublicIpAddress: True + # Uncomment if you would like to use Spot instances (subject to unexpected termination) + # SpotPrice: !Ref SpotPrice + SecurityGroups: + - !Ref SecurityGroup + IamInstanceProfile: !Ref ECSInstanceProfile + UserData: + "Fn::Base64": !Sub | + #!/bin/bash + yum install -y aws-cfn-bootstrap hibagent rsync awscli + yum update -y + service amazon-ssm-agent restart + + # determine if we have an NVMe SSD attached + find /dev/nvme1 + if [ $? -eq 0 ] + then + mount_point=/var/lib/docker + + # copy existing files from mount point + service docker stop + echo 'DOCKER_STORAGE_OPTIONS="--storage-driver overlay2"' > /etc/sysconfig/docker-storage + mkdir -p /tmp$mount_point + rsync -val $mount_point/ /tmp/$mount_point/ + + # make a new filesystem and mount it + mkfs -t ext4 /dev/nvme1n1 + mkdir -p $mount_point + mount -t ext4 -o noatime /dev/nvme1n1 $mount_point + + # Copy files back to new mount point + rsync -val /tmp/$mount_point/ $mount_point/ + rm -rf /tmp$mount_point + service docker start + + # Make raid appear on reboot + echo >> /etc/fstab + echo "/dev/nvme1n1 $mount_point ext4 noatime 0 0" | tee -a /etc/fstab + fi + + # Set Linux traffic control to limit outbound bandwidth usage of peering + #tc qdisc add dev eth0 root handle 1:0 htb default 1 + #tc class add dev eth0 parent 1:0 classid 1:10 htb rate ${Bandwidth}kbit ceil {BandwidthCeiling}kbit prio 0 + #tc filter add dev eth0 protocol ip parent 1:0 prio 1 u32 match ip dport 30303 0xffff flowid 1:10 + + /opt/aws/bin/cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource ECSLaunchConfiguration + /opt/aws/bin/cfn-signal -e $? --region ${AWS::Region} --stack ${AWS::StackName} --resource ECSAutoScalingGroup + /usr/bin/enable-ec2-spot-hibernation + + # Attach an EIP from the pool of available EIPs in scope "vpc" + alloc=`aws ec2 describe-addresses --region ${AWS::Region} --output text | grep -v eni | head -1 | cut -f 2` + instanceid=`curl --silent 169.254.169.254/latest/meta-data/instance-id` + aws ec2 associate-address --region ${AWS::Region} --allocation-id $alloc --instance-id $instanceid + echo "ECS_ENGINE_TASK_CLEANUP_WAIT_DURATION=5m" >> /etc/ecs/ecs.config + + reboot + + Metadata: + AWS::CloudFormation::Init: + config: + packages: + yum: + awslogs: [] + + commands: + 01_add_instance_to_cluster: + command: !Sub echo ECS_CLUSTER=${Cluster} >> /etc/ecs/ecs.config + files: + "/etc/cfn/cfn-hup.conf": + mode: 000400 + owner: root + group: root + content: !Sub | + [main] + stack=${AWS::StackId} + region=${AWS::Region} + + "/etc/cfn/hooks.d/cfn-auto-reloader.conf": + content: !Sub | + [cfn-auto-reloader-hook] + triggers=post.update + path=Resources.ECSLaunchConfiguration.Metadata.AWS::CloudFormation::Init + action=/opt/aws/bin/cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource ECSLaunchConfiguration + + services: + sysvinit: + cfn-hup: + enabled: true + ensureRunning: true + files: + - /etc/cfn/cfn-hup.conf + - /etc/cfn/hooks.d/cfn-auto-reloader.conf + + NodeLoadBalancer: + Type: AWS::ElasticLoadBalancingV2::LoadBalancer + Properties: + Name: !Sub ${AWS::StackName}-node-NLB + Type: network + Scheme: internal + Subnets: !Ref Subnets + Tags: + - Key: Name + Value: !Sub ${AWS::StackName}-node-NLB + + NodeTargetGroup: + Type: AWS::ElasticLoadBalancingV2::TargetGroup + DependsOn: NodeLoadBalancer + Properties: + VpcId: !Ref VPC + Port: !Ref RpcPort + Protocol: TCP + TargetGroupAttributes: + - Key: deregistration_delay.timeout_seconds + Value: 120 + + NodeListener: + Type: AWS::ElasticLoadBalancingV2::Listener + Properties: + DefaultActions: + - Type: forward + TargetGroupArn: !Ref NodeTargetGroup + LoadBalancerArn: !Ref NodeLoadBalancer + Port: !Ref RpcPort + Protocol: TCP + + NodeWsTargetGroup: + Type: AWS::ElasticLoadBalancingV2::TargetGroup + DependsOn: NodeLoadBalancer + Properties: + VpcId: !Ref VPC + Port: !Ref WsPort + Protocol: TCP + TargetGroupAttributes: + - Key: deregistration_delay.timeout_seconds + Value: 120 + + NodeWsListener: + Type: AWS::ElasticLoadBalancingV2::Listener + Properties: + DefaultActions: + - Type: forward + TargetGroupArn: !Ref NodeWsTargetGroup + LoadBalancerArn: !Ref NodeLoadBalancer + Port: !Ref WsPort + Protocol: TCP + + # This IAM Role is attached to all of the ECS hosts. It is based on the default role + # published here: + # http://docs.aws.amazon.com/AmazonECS/latest/developerguide/instance_IAM_role.html + # + # You can add other IAM policy statements here to allow access from your ECS hosts + # to other AWS services. + + ECSRole: + Type: AWS::IAM::Role + Properties: + Path: / + RoleName: !Sub ${AWS::StackName}-ECSRole-${AWS::Region} + AssumeRolePolicyDocument: | + { + "Statement": [{ + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ec2.amazonaws.com" + } + }] + } + Policies: + - PolicyName: ecs-service + PolicyDocument: | + { + "Statement": [{ + "Effect": "Allow", + "Action": [ + "ecs:CreateCluster", + "ecs:DeregisterContainerInstance", + "ecs:DiscoverPollEndpoint", + "ecs:Poll", + "ecs:RegisterContainerInstance", + "ecs:StartTelemetrySession", + "ecs:Submit*", + "logs:CreateLogStream", + "logs:PutLogEvents", + "ecr:BatchCheckLayerAvailability", + "ecr:BatchGetImage", + "ecr:GetDownloadUrlForLayer", + "ecr:GetAuthorizationToken", + "ssm:DescribeAssociation", + "ssm:GetDeployablePatchSnapshotForInstance", + "ssm:GetDocument", + "ssm:GetManifest", + "ssm:GetParameters", + "ssm:ListAssociations", + "ssm:ListInstanceAssociations", + "ssm:PutInventory", + "ssm:PutComplianceItems", + "ssm:PutConfigurePackageResult", + "ssm:PutParameter", + "ssm:UpdateAssociationStatus", + "ssm:UpdateInstanceAssociationStatus", + "ssm:UpdateInstanceInformation", + "ec2messages:AcknowledgeMessage", + "ec2messages:DeleteMessage", + "ec2messages:FailMessage", + "ec2messages:GetEndpoint", + "ec2messages:GetMessages", + "ec2messages:SendReply", + "cloudwatch:PutMetricData", + "ec2:DescribeInstanceStatus", + "ds:CreateComputer", + "ds:DescribeDirectories", + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:DescribeLogGroups", + "logs:DescribeLogStreams", + "logs:PutLogEvents", + "s3:PutObject", + "s3:GetObject", + "s3:DeleteObject", + "s3:AbortMultipartUpload", + "s3:ListMultipartUploadParts", + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "firehose:PutRecord", + "firehose:PutRecordBatch", + "ec2:DescribeAddresses", + "ec2:DescribeInstances", + "ec2:AssociateAddress" + ], + "Resource": "*" + }] + } + + ECSInstanceProfile: + Type: AWS::IAM::InstanceProfile + Properties: + Path: / + Roles: + - !Ref ECSRole + + ECSServiceAutoScalingRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + Action: + - 'sts:AssumeRole' + Effect: Allow + Principal: + Service: + - application-autoscaling.amazonaws.com + Path: / + Policies: + - PolicyName: ecs-service-autoscaling + PolicyDocument: + Statement: + Effect: Allow + Action: + - application-autoscaling:* + - cloudwatch:DescribeAlarms + - cloudwatch:PutMetricAlarm + - ecs:DescribeServices + - ecs:UpdateService + Resource: "*" + + NodeTaskExecutionRole: + Type: AWS::IAM::Role + Properties: + Path: / + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Action: sts:AssumeRole + Effect: Allow + Principal: + Service: ecs-tasks.amazonaws.com + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy + Policies: + - PolicyName: !Sub ecs-task-S3-${AWS::StackName} + PolicyDocument: + Version: 2012-10-17 + Statement: + - + Effect: Allow + Action: + - "s3:Get*" + - "s3:List*" + Resource: + - Fn::Join: + - "" + - + - "arn:aws:s3:::" + - !FindInMap + - RegionMap + - !Ref 'AWS::Region' + - !Ref Network + + NodeLogGroup: + Type: AWS::Logs::LogGroup + Properties: + LogGroupName: !Sub /ecs/${AWS::StackName}-node + RetentionInDays: 14 + + NodeECSService: + Type: AWS::ECS::Service + DependsOn: NodeListener + Properties: + Cluster: !Ref Cluster + DesiredCount: !Ref NodeDesiredCount + HealthCheckGracePeriodSeconds: 3600 + TaskDefinition: !Ref NodeTaskDefinition + LaunchType: EC2 + DeploymentConfiguration: + MaximumPercent: 150 + MinimumHealthyPercent: 50 + LoadBalancers: + - ContainerName: !Ref NodeTaskName + ContainerPort: !Ref RpcPort + TargetGroupArn: !Ref NodeTargetGroup + - ContainerName: !Ref NodeTaskName + ContainerPort: !Ref WsPort + TargetGroupArn: !Ref NodeWsTargetGroup + + NodeTaskDefinition: + Type: AWS::ECS::TaskDefinition + Properties: + Family: !Ref NodeTaskName + RequiresCompatibilities: + - EC2 + NetworkMode: host + ExecutionRoleArn: !Ref NodeTaskExecutionRole + ContainerDefinitions: + - Name: !Ref NodeTaskName + Image: !Sub ${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${NodeRepository} + Essential: true + MemoryReservation: !Ref MemoryLimit + Environment: + - Name: "region" + Value: !Ref AWS::Region + - Name: "network" + Value: !Ref Network + - Name: "syncmode" + Value: !Ref SyncMode + - Name: "connections" + Value: !Ref Connections + - Name: "rpcport" + Value: !Ref RpcPort + - Name: "wsport" + Value: !Ref WsPort + - Name: "netport" + Value: !Ref NetPort + - Name: "chainbucket" + Value: !FindInMap + - RegionMap + - !Ref 'AWS::Region' + - !Ref Network + - Name: "s3key" + Value: node + PortMappings: + - ContainerPort: !Ref RpcPort + - ContainerPort: !Ref WsPort + - ContainerPort: !Ref NetPort + LogConfiguration: + LogDriver: awslogs + Options: + awslogs-region: !Ref AWS::Region + awslogs-group: !Ref NodeLogGroup + awslogs-stream-prefix: !Ref AWS::StackName + #HealthCheck: + # Command: + # - CMD-SHELL + # - '[ `echo "eth.syncing.highestBlock - eth.syncing.currentBlock"|geth attach|head -10|tail -1` -lt 200 ] || exit 1' + # Interval: 300 + # Timeout: 60 + # Retries: 10 + # StartPeriod: 300 + +# CodePipeline Resources + + NodeRepository: + Type: AWS::ECR::Repository + + NodeCodeBuildServiceRole: + Type: AWS::IAM::Role + Properties: + Path: / + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: codebuild.amazonaws.com + Action: sts:AssumeRole + Policies: + - PolicyName: root + PolicyDocument: + Version: 2012-10-17 + Statement: + - Resource: "*" + Effect: Allow + Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents + - ecr:GetAuthorizationToken + - Resource: !Sub arn:aws:s3:::${NodeArtifactBucket}/* + Effect: Allow + Action: + - s3:GetObject + - s3:PutObject + - s3:GetObjectVersion + - Resource: !Sub arn:aws:ecr:${AWS::Region}:${AWS::AccountId}:repository/${NodeRepository} + Effect: Allow + Action: + - ecr:GetDownloadUrlForLayer + - ecr:BatchGetImage + - ecr:BatchCheckLayerAvailability + - ecr:PutImage + - ecr:InitiateLayerUpload + - ecr:UploadLayerPart + - ecr:CompleteLayerUpload + + NodeCodePipelineServiceRole: + Type: AWS::IAM::Role + Properties: + Path: / + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: codepipeline.amazonaws.com + Action: sts:AssumeRole + Policies: + - PolicyName: root + PolicyDocument: + Version: 2012-10-17 + Statement: + - Resource: + - !Sub arn:aws:s3:::${NodeArtifactBucket}/* + Effect: Allow + Action: + - s3:PutObject + - s3:GetObject + - s3:GetObjectVersion + - s3:GetBucketVersioning + - Resource: "*" + Effect: Allow + Action: + - ecs:DescribeServices + - ecs:DescribeTaskDefinition + - ecs:DescribeTasks + - ecs:ListTasks + - ecs:RegisterTaskDefinition + - ecs:UpdateService + - codebuild:StartBuild + - codebuild:BatchGetBuilds + - iam:PassRole + + NodeArtifactBucket: + Type: AWS::S3::Bucket + + NodeCodeBuildProject: + Type: AWS::CodeBuild::Project + Properties: + Artifacts: + Type: CODEPIPELINE + Source: + Type: CODEPIPELINE + BuildSpec: | + version: 0.2 + phases: + install: + runtime-versions: + docker: 19 + pre_build: + commands: + - $(aws ecr get-login --no-include-email) + - TAG="$(echo $CODEBUILD_RESOLVED_SOURCE_VERSION | head -c 8)" + - IMAGE_URI="${REPOSITORY_URI}:${TAG}" + - cp infra/Dockerfile.node ./Dockerfile + build: + commands: + - docker build --tag "$IMAGE_URI" . + - docker build --tag "${REPOSITORY_URI}:latest" . + post_build: + commands: + - docker push "$IMAGE_URI" + - docker push "${REPOSITORY_URI}:latest" + - printf '[{"name":"mev-geth-node","imageUri":"%s"}]' "$IMAGE_URI" > images.json + artifacts: + files: images.json + Environment: + ComputeType: BUILD_GENERAL1_LARGE + Image: aws/codebuild/amazonlinux2-aarch64-standard:1.0 + Type: ARM_CONTAINER + PrivilegedMode: true + EnvironmentVariables: + - Name: AWS_DEFAULT_REGION + Value: !Ref AWS::Region + - Name: REPOSITORY_URI + Value: !Sub ${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${NodeRepository} + Cache: + Type: S3 + Location: !Sub ${NodeArtifactBucket}/buildcache + Name: !Sub ${AWS::StackName}-node + ServiceRole: !Ref NodeCodeBuildServiceRole + + NodePipeline: + Type: AWS::CodePipeline::Pipeline + Properties: + RoleArn: !GetAtt NodeCodePipelineServiceRole.Arn + ArtifactStore: + Type: S3 + Location: !Ref NodeArtifactBucket + Stages: + - Name: Source + Actions: + - Name: App + ActionTypeId: + Category: Source + Owner: ThirdParty + Version: 1 + Provider: GitHub + Configuration: + Owner: !Ref GitHubUser + Repo: !Ref NodeGitHubRepo + Branch: !Ref NodeGitHubBranch + OAuthToken: !Ref GitHubToken + OutputArtifacts: + - Name: App + RunOrder: 1 + - Name: Build + Actions: + - Name: Build + ActionTypeId: + Category: Build + Owner: AWS + Version: 1 + Provider: CodeBuild + Configuration: + ProjectName: !Ref NodeCodeBuildProject + InputArtifacts: + - Name: App + OutputArtifacts: + - Name: BuildOutput + RunOrder: 1 + - Name: Deploy + Actions: + - Name: Deploy + ActionTypeId: + Category: Deploy + Owner: AWS + Version: 1 + Provider: ECS + Configuration: + ClusterName: !Ref Cluster + ServiceName: !Ref NodeECSService + FileName: images.json + InputArtifacts: + - Name: BuildOutput + RunOrder: 1 + +# SNS Resources + + SNSTopic: + Type: AWS::SNS::Topic + Properties: + DisplayName: String + Subscription: + - + Endpoint: !Ref SNSSubscriptionEndpoint + Protocol: !Ref SNSSubscriptionProtocol + TopicName: !Ref AWS::StackName + +# CloudWatch Resources + + CPUAlarm: + Type: AWS::CloudWatch::Alarm + Properties: + AlarmName: !Sub ${AWS::StackName} average CPU utilization greater than threshold. + AlarmDescription: Alarm if CPU utilization is greater than threshold. + Namespace: AWS/ECS + MetricName: CPUUtilization + Dimensions: + - Name: ClusterName + Value: !Ref Cluster + Statistic: Average + Period: '60' + EvaluationPeriods: '3' + Threshold: !Ref CPUAlarmThreshold + ComparisonOperator: GreaterThanThreshold + AlarmActions: + - Ref: SNSTopic + OKActions: + - Ref: SNSTopic + + MemoryAlarm: + Type: AWS::CloudWatch::Alarm + Properties: + AlarmName: !Sub ${AWS::StackName} average memory utilization greater than threshold. + AlarmDescription: Alarm if memory utilization is greater than threshold. + Namespace: AWS/ECS + MetricName: MemoryUtilization + Dimensions: + - Name: ClusterName + Value: !Ref Cluster + Statistic: Average + Period: '60' + EvaluationPeriods: '3' + Threshold: !Ref MemoryAlarmThreshold + ComparisonOperator: GreaterThanThreshold + AlarmActions: + - Ref: SNSTopic + OKActions: + - Ref: SNSTopic + + HealthyHostAlarm: + Type: 'AWS::CloudWatch::Alarm' + Properties: + AlarmName: !Sub ${AWS::StackName} alarm no healthy hosts connected to ELB. + AlarmDescription: Alarm if no healthy hosts connected to ELB. + MetricName: HealthyHostCount + Namespace: AWS/NetworkELB + Statistic: Average + Period: '60' + EvaluationPeriods: '3' + Threshold: '1' + ComparisonOperator: LessThanThreshold + Dimensions: + - Name: TargetGroup + Value: !GetAtt NodeTargetGroup.TargetGroupFullName + - Name: LoadBalancer + Value: !GetAtt NodeLoadBalancer.LoadBalancerFullName + AlarmActions: + - Ref: SNSTopic + OKActions: + - Ref: SNSTopic + +Outputs: + ClusterName: + Value: !Ref Cluster + NodeService: + Value: !Ref NodeECSService + NodePipelineUrl: + Value: !Sub https://console.aws.amazon.com/codepipeline/home?region=${AWS::Region}#/view/${NodePipeline} + NodeTargetGroup: + Value: !Ref NodeTargetGroup + NodeServiceUrl: + Description: URL of the load balancer for the node service. + Value: !Sub http://${NodeLoadBalancer.DNSName} diff --git a/infra/mev-geth-nodes-x86-64.yaml b/infra/mev-geth-nodes-x86-64.yaml new file mode 100644 index 000000000000..bf7a196caa52 --- /dev/null +++ b/infra/mev-geth-nodes-x86-64.yaml @@ -0,0 +1,972 @@ +--- +AWSTemplateFormatVersion: 2010-09-09 + +Description: > + This template creates an automated continuous deployment pipeline to Amazon Elastic Container Service (ECS) + Created by Luke Youngblood, luke@blockscale.net + +Parameters: + # GitHub Parameters + + GitHubUser: + Type: String + Default: lyoungblood + Description: Your team or username on GitHub. + + NodeGitHubRepo: + Type: String + Default: mev-geth + Description: The repo name of the node service. + + NodeGitHubBranch: + Type: String + Default: master + Description: The branch of the node repo to continuously deploy. + + GitHubToken: + Type: String + NoEcho: true + Description: > + Token for the team or user specified above. (https://github.com/settings/tokens) + + # VPC Parameters + + VPC: + Type: AWS::EC2::VPC::Id + + Subnets: + Type: List + + VpcCIDR: + Type: String + Default: 172.31.0.0/16 + + # ECS Parameters + + InstanceType: + Type: String + Default: i3en.large + + MemoryLimit: + Type: Number + Default: 6144 + + KeyPair: + Type: AWS::EC2::KeyPair::KeyName + + SpotPrice: + Type: Number + Default: 0.0904 + + ClusterSize: + Type: Number + Default: 5 + + Bandwidth: + Type: Number + Default: 2048 + + BandwidthCeiling: + Type: Number + Default: 4096 + + NodeDesiredCount: + Type: Number + Default: 0 + + NodeTaskName: + Type: String + Default: mev-geth-node + + ECSAMI: + Type: AWS::SSM::Parameter::Value + Default: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id + + # SNS Parameters + + SNSSubscriptionEndpoint: + Type: String + Default: https://events.pagerduty.com/integration/44cbdb66f22b4f3caf5dd15741c7eb17/enqueue + + SNSSubscriptionProtocol: + Type: String + Default: HTTPS + + # CloudWatch Alarm Parameters + + CPUAlarmThreshold: + Type: Number + Default: 80 + + MemoryAlarmThreshold: + Type: Number + Default: 80 + + # Mev-Geth Parameters + + Network: + Type: String + Default: mainnet + AllowedValues: + - mainnet + - goerli + + SyncMode: + Type: String + Default: fast + AllowedValues: + - full + - fast + - light + + Connections: + Type: Number + Default: 50 + + RpcPort: + Type: Number + Default: 8545 + + WsPort: + Type: Number + Default: 8546 + + NetPort: + Type: Number + Default: 30303 + +Metadata: + AWS::CloudFormation::Interface: + ParameterLabels: + GitHubUser: + default: "User" + NodeGitHubRepo: + default: "Node Repo" + NodeGitHubBranch: + default: "Node Branch" + GitHubToken: + default: "Personal Access Token" + VPC: + default: "Choose which VPC the autoscaling group should be deployed to" + Subnets: + default: "Choose which subnets the autoscaling group should be deployed to" + VpcCIDR: + default: "VPC CIDR Block" + InstanceType: + default: "Which instance type should we use to build the ECS cluster?" + MemoryLimit: + default: "How much memory should be reserved for each task. Set to greater than 50% of instance memory capacity." + KeyPair: + default: "Which keypair should be used to allow SSH to the nodes?" + ClusterSize: + default: "How many ECS hosts do you want to initially deploy?" + SpotPrice: + default: "The maximum spot price to pay for instances - this should normally be set to the on demand price." + Bandwidth: + default: "How much bandwidth, in kb/sec., should be allocated to Ethereum peers (upload) per EC2 instance" + BandwidthCeiling: + default: "How much bandwidth, in kb/sec., should be allocated to Ethereum peers as a ceiling (max. upload)" + NodeDesiredCount: + default: "How many ECS Tasks do you want to initially execute?" + NodeTaskName: + default: "The name of the node ECS Task" + ECSAMI: + default: "The ECS AMI ID populated from SSM." + Network: + default: "The Ethereum network you will be connecting to" + SyncMode: + default: "The synchronization mode that Mev-Geth should use (full, fast, or light)" + Connections: + default: "The number of desired connections on the Mev-Geth node" + RpcPort: + default: "The RPC port used for communication with the local Mev-Geth node" + WsPort: + default: "The Websockets port used for communication with the local Mev-Geth node" + NetPort: + default: "The TCP port used for connectivity to other Ethereum peer nodes" + ParameterGroups: + - Label: + default: GitHub Configuration + Parameters: + - NodeGitHubRepo + - NodeGitHubBranch + - GitHubUser + - GitHubToken + - Label: + default: VPC Configuration + Parameters: + - VPC + - Subnets + - VpcCIDR + - Label: + default: ECS Configuration + Parameters: + - InstanceType + - MemoryLimit + - KeyPair + - SpotPrice + - ClusterSize + - Bandwidth + - BandwidthCeiling + - NodeDesiredCount + - NodeTaskName + - ECSAMI + - Label: + default: Mev-Geth Configuration + Parameters: + - Network + - SyncMode + - Connections + - RpcPort + - WsPort + - NetPort + - Label: + default: PagerDuty Endpoint Configuration + Parameters: + - SNSSubscriptionEndpoint + - SNSSubscriptionProtocol + - Label: + default: CloudWatch Alarms Configuration + Parameters: + - CPUAlarmThreshold + - MemoryAlarmThreshold + +# Mappings + +Mappings: + RegionMap: + us-east-2: + mainnet: mev-geth-updater-fast-chainbucket-17p2xhnhcydlz + goerli: mev-geth-updater-fast-goerli-chainbucket-j6dujg8apbna + #us-west-2: + # mainnet: + # goerli: + +Resources: + # ECS Resources + + Cluster: + Type: AWS::ECS::Cluster + Properties: + ClusterName: !Ref AWS::StackName + + SecurityGroup: + Type: "AWS::EC2::SecurityGroup" + Properties: + GroupDescription: !Sub ${AWS::StackName}-sg + VpcId: !Ref VPC + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 22 + ToPort: 22 + CidrIp: !Ref VpcCIDR + - IpProtocol: tcp + FromPort: !Ref RpcPort + ToPort: !Ref RpcPort + CidrIp: !Ref VpcCIDR + - IpProtocol: tcp + FromPort: !Ref WsPort + ToPort: !Ref WsPort + CidrIp: !Ref VpcCIDR + - IpProtocol: tcp + FromPort: !Ref NetPort + ToPort: !Ref NetPort + CidrIp: 0.0.0.0/0 + - IpProtocol: tcp + FromPort: !Ref NetPort + ToPort: !Ref NetPort + CidrIpv6: ::/0 + - IpProtocol: udp + FromPort: !Ref NetPort + ToPort: !Ref NetPort + CidrIp: 0.0.0.0/0 + - IpProtocol: udp + FromPort: !Ref NetPort + ToPort: !Ref NetPort + CidrIpv6: ::/0 + + ECSAutoScalingGroup: + Type: AWS::AutoScaling::AutoScalingGroup + Properties: + VPCZoneIdentifier: !Ref Subnets + LaunchConfigurationName: !Ref ECSLaunchConfiguration + MinSize: !Ref ClusterSize + MaxSize: !Ref ClusterSize + DesiredCapacity: !Ref ClusterSize + Tags: + - Key: Name + Value: !Sub ${AWS::StackName} ECS host + PropagateAtLaunch: true + CreationPolicy: + ResourceSignal: + Timeout: PT15M + UpdatePolicy: + AutoScalingRollingUpdate: + MinInstancesInService: 2 + MaxBatchSize: 1 + PauseTime: PT15M + SuspendProcesses: + - HealthCheck + - ReplaceUnhealthy + - AZRebalance + - AlarmNotification + - ScheduledActions + WaitOnResourceSignals: true + + ECSLaunchConfiguration: + Type: AWS::AutoScaling::LaunchConfiguration + Properties: + ImageId: !Ref ECSAMI + InstanceType: !Ref InstanceType + KeyName: !Ref KeyPair + AssociatePublicIpAddress: True + # Uncomment if you would like to use Spot instances (subject to unexpected termination) + # SpotPrice: !Ref SpotPrice + SecurityGroups: + - !Ref SecurityGroup + IamInstanceProfile: !Ref ECSInstanceProfile + UserData: + "Fn::Base64": !Sub | + #!/bin/bash + yum install -y aws-cfn-bootstrap hibagent rsync awscli + yum update -y + service amazon-ssm-agent restart + + # determine if we have an NVMe SSD attached + find /dev/nvme1 + if [ $? -eq 0 ] + then + mount_point=/var/lib/docker + + # copy existing files from mount point + service docker stop + echo 'DOCKER_STORAGE_OPTIONS="--storage-driver overlay2"' > /etc/sysconfig/docker-storage + mkdir -p /tmp$mount_point + rsync -val $mount_point/ /tmp/$mount_point/ + + # make a new filesystem and mount it + mkfs -t ext4 /dev/nvme1n1 + mkdir -p $mount_point + mount -t ext4 -o noatime /dev/nvme1n1 $mount_point + + # Copy files back to new mount point + rsync -val /tmp/$mount_point/ $mount_point/ + rm -rf /tmp$mount_point + service docker start + + # Make raid appear on reboot + echo >> /etc/fstab + echo "/dev/nvme1n1 $mount_point ext4 noatime 0 0" | tee -a /etc/fstab + fi + + # Set Linux traffic control to limit outbound bandwidth usage of peering + #tc qdisc add dev eth0 root handle 1:0 htb default 1 + #tc class add dev eth0 parent 1:0 classid 1:10 htb rate ${Bandwidth}kbit ceil {BandwidthCeiling}kbit prio 0 + #tc filter add dev eth0 protocol ip parent 1:0 prio 1 u32 match ip dport 30303 0xffff flowid 1:10 + + /opt/aws/bin/cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource ECSLaunchConfiguration + /opt/aws/bin/cfn-signal -e $? --region ${AWS::Region} --stack ${AWS::StackName} --resource ECSAutoScalingGroup + /usr/bin/enable-ec2-spot-hibernation + + # Attach an EIP from the pool of available EIPs in scope "vpc" + alloc=`aws ec2 describe-addresses --region ${AWS::Region} --output text | grep -v eni | head -1 | cut -f 2` + instanceid=`curl --silent 169.254.169.254/latest/meta-data/instance-id` + aws ec2 associate-address --region ${AWS::Region} --allocation-id $alloc --instance-id $instanceid + echo "ECS_ENGINE_TASK_CLEANUP_WAIT_DURATION=5m" >> /etc/ecs/ecs.config + + reboot + + Metadata: + AWS::CloudFormation::Init: + config: + packages: + yum: + awslogs: [] + + commands: + 01_add_instance_to_cluster: + command: !Sub echo ECS_CLUSTER=${Cluster} >> /etc/ecs/ecs.config + files: + "/etc/cfn/cfn-hup.conf": + mode: 000400 + owner: root + group: root + content: !Sub | + [main] + stack=${AWS::StackId} + region=${AWS::Region} + + "/etc/cfn/hooks.d/cfn-auto-reloader.conf": + content: !Sub | + [cfn-auto-reloader-hook] + triggers=post.update + path=Resources.ECSLaunchConfiguration.Metadata.AWS::CloudFormation::Init + action=/opt/aws/bin/cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource ECSLaunchConfiguration + + services: + sysvinit: + cfn-hup: + enabled: true + ensureRunning: true + files: + - /etc/cfn/cfn-hup.conf + - /etc/cfn/hooks.d/cfn-auto-reloader.conf + + NodeLoadBalancer: + Type: AWS::ElasticLoadBalancingV2::LoadBalancer + Properties: + Name: !Sub ${AWS::StackName}-node-NLB + Type: network + Scheme: internal + Subnets: !Ref Subnets + Tags: + - Key: Name + Value: !Sub ${AWS::StackName}-node-NLB + + NodeTargetGroup: + Type: AWS::ElasticLoadBalancingV2::TargetGroup + DependsOn: NodeLoadBalancer + Properties: + VpcId: !Ref VPC + Port: !Ref RpcPort + Protocol: TCP + TargetGroupAttributes: + - Key: deregistration_delay.timeout_seconds + Value: 120 + + NodeListener: + Type: AWS::ElasticLoadBalancingV2::Listener + Properties: + DefaultActions: + - Type: forward + TargetGroupArn: !Ref NodeTargetGroup + LoadBalancerArn: !Ref NodeLoadBalancer + Port: !Ref RpcPort + Protocol: TCP + + NodeWsTargetGroup: + Type: AWS::ElasticLoadBalancingV2::TargetGroup + DependsOn: NodeLoadBalancer + Properties: + VpcId: !Ref VPC + Port: !Ref WsPort + Protocol: TCP + TargetGroupAttributes: + - Key: deregistration_delay.timeout_seconds + Value: 120 + + NodeWsListener: + Type: AWS::ElasticLoadBalancingV2::Listener + Properties: + DefaultActions: + - Type: forward + TargetGroupArn: !Ref NodeWsTargetGroup + LoadBalancerArn: !Ref NodeLoadBalancer + Port: !Ref WsPort + Protocol: TCP + + # This IAM Role is attached to all of the ECS hosts. It is based on the default role + # published here: + # http://docs.aws.amazon.com/AmazonECS/latest/developerguide/instance_IAM_role.html + # + # You can add other IAM policy statements here to allow access from your ECS hosts + # to other AWS services. + + ECSRole: + Type: AWS::IAM::Role + Properties: + Path: / + RoleName: !Sub ${AWS::StackName}-ECSRole-${AWS::Region} + AssumeRolePolicyDocument: | + { + "Statement": [{ + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ec2.amazonaws.com" + } + }] + } + Policies: + - PolicyName: ecs-service + PolicyDocument: | + { + "Statement": [{ + "Effect": "Allow", + "Action": [ + "ecs:CreateCluster", + "ecs:DeregisterContainerInstance", + "ecs:DiscoverPollEndpoint", + "ecs:Poll", + "ecs:RegisterContainerInstance", + "ecs:StartTelemetrySession", + "ecs:Submit*", + "logs:CreateLogStream", + "logs:PutLogEvents", + "ecr:BatchCheckLayerAvailability", + "ecr:BatchGetImage", + "ecr:GetDownloadUrlForLayer", + "ecr:GetAuthorizationToken", + "ssm:DescribeAssociation", + "ssm:GetDeployablePatchSnapshotForInstance", + "ssm:GetDocument", + "ssm:GetManifest", + "ssm:GetParameters", + "ssm:ListAssociations", + "ssm:ListInstanceAssociations", + "ssm:PutInventory", + "ssm:PutComplianceItems", + "ssm:PutConfigurePackageResult", + "ssm:PutParameter", + "ssm:UpdateAssociationStatus", + "ssm:UpdateInstanceAssociationStatus", + "ssm:UpdateInstanceInformation", + "ec2messages:AcknowledgeMessage", + "ec2messages:DeleteMessage", + "ec2messages:FailMessage", + "ec2messages:GetEndpoint", + "ec2messages:GetMessages", + "ec2messages:SendReply", + "cloudwatch:PutMetricData", + "ec2:DescribeInstanceStatus", + "ds:CreateComputer", + "ds:DescribeDirectories", + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:DescribeLogGroups", + "logs:DescribeLogStreams", + "logs:PutLogEvents", + "s3:PutObject", + "s3:GetObject", + "s3:DeleteObject", + "s3:AbortMultipartUpload", + "s3:ListMultipartUploadParts", + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "firehose:PutRecord", + "firehose:PutRecordBatch", + "ec2:DescribeAddresses", + "ec2:DescribeInstances", + "ec2:AssociateAddress" + ], + "Resource": "*" + }] + } + + ECSInstanceProfile: + Type: AWS::IAM::InstanceProfile + Properties: + Path: / + Roles: + - !Ref ECSRole + + ECSServiceAutoScalingRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + Action: + - "sts:AssumeRole" + Effect: Allow + Principal: + Service: + - application-autoscaling.amazonaws.com + Path: / + Policies: + - PolicyName: ecs-service-autoscaling + PolicyDocument: + Statement: + Effect: Allow + Action: + - application-autoscaling:* + - cloudwatch:DescribeAlarms + - cloudwatch:PutMetricAlarm + - ecs:DescribeServices + - ecs:UpdateService + Resource: "*" + + NodeTaskExecutionRole: + Type: AWS::IAM::Role + Properties: + Path: / + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Action: sts:AssumeRole + Effect: Allow + Principal: + Service: ecs-tasks.amazonaws.com + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy + Policies: + - PolicyName: !Sub ecs-task-S3-${AWS::StackName} + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - "s3:Get*" + - "s3:List*" + Resource: + - Fn::Join: + - "" + - - "arn:aws:s3:::" + - !FindInMap + - RegionMap + - !Ref "AWS::Region" + - !Ref Network + + NodeLogGroup: + Type: AWS::Logs::LogGroup + Properties: + LogGroupName: !Sub /ecs/${AWS::StackName}-node + RetentionInDays: 14 + + NodeECSService: + Type: AWS::ECS::Service + DependsOn: NodeListener + Properties: + Cluster: !Ref Cluster + DesiredCount: !Ref NodeDesiredCount + HealthCheckGracePeriodSeconds: 3600 + TaskDefinition: !Ref NodeTaskDefinition + LaunchType: EC2 + DeploymentConfiguration: + MaximumPercent: 150 + MinimumHealthyPercent: 50 + LoadBalancers: + - ContainerName: !Ref NodeTaskName + ContainerPort: !Ref RpcPort + TargetGroupArn: !Ref NodeTargetGroup + - ContainerName: !Ref NodeTaskName + ContainerPort: !Ref WsPort + TargetGroupArn: !Ref NodeWsTargetGroup + + NodeTaskDefinition: + Type: AWS::ECS::TaskDefinition + Properties: + Family: !Ref NodeTaskName + RequiresCompatibilities: + - EC2 + NetworkMode: host + ExecutionRoleArn: !Ref NodeTaskExecutionRole + ContainerDefinitions: + - Name: !Ref NodeTaskName + Image: !Sub ${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${NodeRepository} + Essential: true + MemoryReservation: !Ref MemoryLimit + Environment: + - Name: "region" + Value: !Ref AWS::Region + - Name: "network" + Value: !Ref Network + - Name: "syncmode" + Value: !Ref SyncMode + - Name: "connections" + Value: !Ref Connections + - Name: "rpcport" + Value: !Ref RpcPort + - Name: "wsport" + Value: !Ref WsPort + - Name: "netport" + Value: !Ref NetPort + - Name: "chainbucket" + Value: !FindInMap + - RegionMap + - !Ref "AWS::Region" + - !Ref Network + - Name: "s3key" + Value: node + PortMappings: + - ContainerPort: !Ref RpcPort + - ContainerPort: !Ref WsPort + - ContainerPort: !Ref NetPort + LogConfiguration: + LogDriver: awslogs + Options: + awslogs-region: !Ref AWS::Region + awslogs-group: !Ref NodeLogGroup + awslogs-stream-prefix: !Ref AWS::StackName + #HealthCheck: + # Command: + # - CMD-SHELL + # - '[ `echo "eth.syncing.highestBlock - eth.syncing.currentBlock"|geth attach|head -10|tail -1` -lt 200 ] || exit 1' + # Interval: 300 + # Timeout: 60 + # Retries: 10 + # StartPeriod: 300 + + # CodePipeline Resources + + NodeRepository: + Type: AWS::ECR::Repository + + NodeCodeBuildServiceRole: + Type: AWS::IAM::Role + Properties: + Path: / + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: codebuild.amazonaws.com + Action: sts:AssumeRole + Policies: + - PolicyName: root + PolicyDocument: + Version: 2012-10-17 + Statement: + - Resource: "*" + Effect: Allow + Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents + - ecr:GetAuthorizationToken + - Resource: !Sub arn:aws:s3:::${NodeArtifactBucket}/* + Effect: Allow + Action: + - s3:GetObject + - s3:PutObject + - s3:GetObjectVersion + - Resource: !Sub arn:aws:ecr:${AWS::Region}:${AWS::AccountId}:repository/${NodeRepository} + Effect: Allow + Action: + - ecr:GetDownloadUrlForLayer + - ecr:BatchGetImage + - ecr:BatchCheckLayerAvailability + - ecr:PutImage + - ecr:InitiateLayerUpload + - ecr:UploadLayerPart + - ecr:CompleteLayerUpload + + NodeCodePipelineServiceRole: + Type: AWS::IAM::Role + Properties: + Path: / + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: codepipeline.amazonaws.com + Action: sts:AssumeRole + Policies: + - PolicyName: root + PolicyDocument: + Version: 2012-10-17 + Statement: + - Resource: + - !Sub arn:aws:s3:::${NodeArtifactBucket}/* + Effect: Allow + Action: + - s3:PutObject + - s3:GetObject + - s3:GetObjectVersion + - s3:GetBucketVersioning + - Resource: "*" + Effect: Allow + Action: + - ecs:DescribeServices + - ecs:DescribeTaskDefinition + - ecs:DescribeTasks + - ecs:ListTasks + - ecs:RegisterTaskDefinition + - ecs:UpdateService + - codebuild:StartBuild + - codebuild:BatchGetBuilds + - iam:PassRole + + NodeArtifactBucket: + Type: AWS::S3::Bucket + + NodeCodeBuildProject: + Type: AWS::CodeBuild::Project + Properties: + Artifacts: + Type: CODEPIPELINE + Source: + Type: CODEPIPELINE + BuildSpec: | + version: 0.2 + phases: + install: + runtime-versions: + docker: 19 + pre_build: + commands: + - $(aws ecr get-login --no-include-email) + - TAG="$(echo $CODEBUILD_RESOLVED_SOURCE_VERSION | head -c 8)" + - IMAGE_URI="${REPOSITORY_URI}:${TAG}" + - cp infra/Dockerfile.node ./Dockerfile + build: + commands: + - docker build --tag "$IMAGE_URI" . + - docker build --tag "${REPOSITORY_URI}:latest" . + post_build: + commands: + - docker push "$IMAGE_URI" + - docker push "${REPOSITORY_URI}:latest" + - printf '[{"name":"mev-geth-node","imageUri":"%s"}]' "$IMAGE_URI" > images.json + artifacts: + files: images.json + Environment: + ComputeType: BUILD_GENERAL1_SMALL + Image: aws/codebuild/docker:17.09.0 + Type: LINUX_CONTAINER + PrivilegedMode: true + EnvironmentVariables: + - Name: AWS_DEFAULT_REGION + Value: !Ref AWS::Region + - Name: REPOSITORY_URI + Value: !Sub ${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${NodeRepository} + Cache: + Type: S3 + Location: !Sub ${NodeArtifactBucket}/buildcache + Name: !Sub ${AWS::StackName}-node + ServiceRole: !Ref NodeCodeBuildServiceRole + + NodePipeline: + Type: AWS::CodePipeline::Pipeline + Properties: + RoleArn: !GetAtt NodeCodePipelineServiceRole.Arn + ArtifactStore: + Type: S3 + Location: !Ref NodeArtifactBucket + Stages: + - Name: Source + Actions: + - Name: App + ActionTypeId: + Category: Source + Owner: ThirdParty + Version: 1 + Provider: GitHub + Configuration: + Owner: !Ref GitHubUser + Repo: !Ref NodeGitHubRepo + Branch: !Ref NodeGitHubBranch + OAuthToken: !Ref GitHubToken + OutputArtifacts: + - Name: App + RunOrder: 1 + - Name: Build + Actions: + - Name: Build + ActionTypeId: + Category: Build + Owner: AWS + Version: 1 + Provider: CodeBuild + Configuration: + ProjectName: !Ref NodeCodeBuildProject + InputArtifacts: + - Name: App + OutputArtifacts: + - Name: BuildOutput + RunOrder: 1 + - Name: Deploy + Actions: + - Name: Deploy + ActionTypeId: + Category: Deploy + Owner: AWS + Version: 1 + Provider: ECS + Configuration: + ClusterName: !Ref Cluster + ServiceName: !Ref NodeECSService + FileName: images.json + InputArtifacts: + - Name: BuildOutput + RunOrder: 1 + + # SNS Resources + + SNSTopic: + Type: AWS::SNS::Topic + Properties: + DisplayName: String + Subscription: + - Endpoint: !Ref SNSSubscriptionEndpoint + Protocol: !Ref SNSSubscriptionProtocol + TopicName: !Ref AWS::StackName + + # CloudWatch Resources + + CPUAlarm: + Type: AWS::CloudWatch::Alarm + Properties: + AlarmName: !Sub ${AWS::StackName} average CPU utilization greater than threshold. + AlarmDescription: Alarm if CPU utilization is greater than threshold. + Namespace: AWS/ECS + MetricName: CPUUtilization + Dimensions: + - Name: ClusterName + Value: !Ref Cluster + Statistic: Average + Period: "60" + EvaluationPeriods: "3" + Threshold: !Ref CPUAlarmThreshold + ComparisonOperator: GreaterThanThreshold + AlarmActions: + - Ref: SNSTopic + OKActions: + - Ref: SNSTopic + + MemoryAlarm: + Type: AWS::CloudWatch::Alarm + Properties: + AlarmName: !Sub ${AWS::StackName} average memory utilization greater than threshold. + AlarmDescription: Alarm if memory utilization is greater than threshold. + Namespace: AWS/ECS + MetricName: MemoryUtilization + Dimensions: + - Name: ClusterName + Value: !Ref Cluster + Statistic: Average + Period: "60" + EvaluationPeriods: "3" + Threshold: !Ref MemoryAlarmThreshold + ComparisonOperator: GreaterThanThreshold + AlarmActions: + - Ref: SNSTopic + OKActions: + - Ref: SNSTopic + + HealthyHostAlarm: + Type: "AWS::CloudWatch::Alarm" + Properties: + AlarmName: !Sub ${AWS::StackName} alarm no healthy hosts connected to ELB. + AlarmDescription: Alarm if no healthy hosts connected to ELB. + MetricName: HealthyHostCount + Namespace: AWS/NetworkELB + Statistic: Average + Period: "60" + EvaluationPeriods: "3" + Threshold: "1" + ComparisonOperator: LessThanThreshold + Dimensions: + - Name: TargetGroup + Value: !GetAtt NodeTargetGroup.TargetGroupFullName + - Name: LoadBalancer + Value: !GetAtt NodeLoadBalancer.LoadBalancerFullName + AlarmActions: + - Ref: SNSTopic + OKActions: + - Ref: SNSTopic + +Outputs: + ClusterName: + Value: !Ref Cluster + NodeService: + Value: !Ref NodeECSService + NodePipelineUrl: + Value: !Sub https://console.aws.amazon.com/codepipeline/home?region=${AWS::Region}#/view/${NodePipeline} + NodeTargetGroup: + Value: !Ref NodeTargetGroup + NodeServiceUrl: + Description: URL of the load balancer for the node service. + Value: !Sub http://${NodeLoadBalancer.DNSName} diff --git a/infra/mev-geth-updater-arm64.yaml b/infra/mev-geth-updater-arm64.yaml new file mode 100644 index 000000000000..ad81ece1b034 --- /dev/null +++ b/infra/mev-geth-updater-arm64.yaml @@ -0,0 +1,749 @@ +--- +AWSTemplateFormatVersion: 2010-09-09 + +Description: > + This template creates an automated continuous deployment pipeline to Amazon Elastic Container Service (ECS) + Created by Luke Youngblood, luke@blockscale.net + +Parameters: + +# GitHub Parameters + + GitHubUser: + Type: String + Default: lyoungblood + Description: Your team or username on GitHub. + + GitHubRepo: + Type: String + Default: mev-geth + Description: The repo name of the baker service. + + GitHubBranch: + Type: String + Default: master + Description: The branch of the repo to continuously deploy. + + GitHubToken: + Type: String + NoEcho: true + Description: > + Token for the team or user specified above. (https://github.com/settings/tokens) + +# VPC Parameters + + VPC: + Type: AWS::EC2::VPC::Id + + Subnets: + Type: List + + VpcCIDR: + Type: String + Default: 172.31.0.0/16 + +# ECS Parameters + + InstanceType: + Type: String + Default: m6gd.large + + KeyPair: + Type: AWS::EC2::KeyPair::KeyName + + ClusterSize: + Type: Number + Default: 1 + + DesiredCount: + Type: Number + Default: 0 + + TaskName: + Type: String + Default: mev-geth-updater + + ECSAMI: + Type: AWS::SSM::Parameter::Value + Default: /aws/service/ecs/optimized-ami/amazon-linux-2/arm64/recommended/image_id + +# Mev-Geth Parameters + + Network: + Type: String + Default: mainnet + AllowedValues: + - mainnet + - goerli + + SyncMode: + Type: String + Default: fast + AllowedValues: + - full + - fast + - light + + Connections: + Type: Number + Default: 50 + + NetPort: + Type: Number + Default: 30303 + +Metadata: + + AWS::CloudFormation::Interface: + ParameterLabels: + GitHubUser: + default: "User" + GitHubRepo: + default: "Mev-Geth GitHub Repository" + GitHubBranch: + default: "Branch in GitHub repository" + GitHubToken: + default: "Personal Access Token" + VPC: + default: "Choose which VPC the autoscaling group should be deployed to" + Subnets: + default: "Choose which subnets the autoscaling group should be deployed to" + VpcCIDR: + default: "VPC CIDR Block" + InstanceType: + default: "Which instance type should we use to build the ECS cluster?" + KeyPair: + default: "Which keypair should be used for access to the ECS cluster?" + ClusterSize: + default: "How many ECS hosts do you want to initially deploy?" + DesiredCount: + default: "How many Updater tasks do you want to initially execute?" + TaskName: + default: "The name of the Updater ECS Task" + ECSAMI: + default: "The ECS AMI ID populated from SSM." + Network: + default: "The network the Mev-Geth node should join" + SyncMode: + default: "The synchronization mode that Mev-Geth should use (full, fast, or light)" + Connections: + default: "The number of connections the Mev-Geth node should be configured with" + NetPort: + default: "The TCP/UDP port used for Mev-Geth connectivity to other Ethereum peer nodes" + ParameterGroups: + - Label: + default: GitHub Configuration + Parameters: + - GitHubRepo + - GitHubBranch + - GitHubUser + - GitHubToken + - Label: + default: VPC Configuration + Parameters: + - VPC + - Subnets + - VpcCIDR + - Label: + default: ECS Configuration + Parameters: + - InstanceType + - KeyPair + - ClusterSize + - DesiredCount + - TaskName + - ECSAMI + - Label: + default: Mev-Geth Configuration + Parameters: + - Network + - SyncMode + - Connections + - NetPort + +Resources: + +# ECS Resources + + ChainBucket: + Type: AWS::S3::Bucket + + ChainBucketPolicy: + Type: AWS::S3::BucketPolicy + Properties: + Bucket: !Ref ChainBucket + PolicyDocument: + Statement: + - + Action: + - s3:GetObject + - s3:ListBucket + Effect: Allow + Resource: + - Fn::Join: + - "" + - + - "arn:aws:s3:::" + - + Ref: "ChainBucket" + - "/*" + - Fn::Join: + - "" + - + - "arn:aws:s3:::" + - + Ref: "ChainBucket" + Principal: + AWS: "*" + + Cluster: + Type: AWS::ECS::Cluster + Properties: + ClusterName: !Ref AWS::StackName + + SecurityGroup: + Type: "AWS::EC2::SecurityGroup" + Properties: + GroupDescription: !Sub ${AWS::StackName}-sg + VpcId: !Ref VPC + Tags: + - + Key: Name + Value: !Sub ${AWS::StackName}-sg + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 22 + ToPort: 22 + CidrIp: !Ref VpcCIDR + - IpProtocol: tcp + FromPort: !Ref NetPort + ToPort: !Ref NetPort + CidrIp: 0.0.0.0/0 + - IpProtocol: tcp + FromPort: !Ref NetPort + ToPort: !Ref NetPort + CidrIpv6: ::/0 + - IpProtocol: udp + FromPort: !Ref NetPort + ToPort: !Ref NetPort + CidrIp: 0.0.0.0/0 + - IpProtocol: udp + FromPort: !Ref NetPort + ToPort: !Ref NetPort + CidrIpv6: ::/0 + + ECSAutoScalingGroup: + Type: AWS::AutoScaling::AutoScalingGroup + Properties: + VPCZoneIdentifier: !Ref Subnets + LaunchConfigurationName: !Ref ECSLaunchConfiguration + MinSize: !Ref ClusterSize + MaxSize: !Ref ClusterSize + DesiredCapacity: !Ref ClusterSize + Tags: + - Key: Name + Value: !Sub ${AWS::StackName} ECS host + PropagateAtLaunch: true + CreationPolicy: + ResourceSignal: + Timeout: PT15M + UpdatePolicy: + AutoScalingRollingUpdate: + MinInstancesInService: 0 + MaxBatchSize: 1 + PauseTime: PT15M + SuspendProcesses: + - HealthCheck + - ReplaceUnhealthy + - AZRebalance + - AlarmNotification + - ScheduledActions + WaitOnResourceSignals: true + + ECSLaunchConfiguration: + Type: AWS::AutoScaling::LaunchConfiguration + Properties: + ImageId: !Ref ECSAMI + InstanceType: !Ref InstanceType + KeyName: !Ref KeyPair + SecurityGroups: + - !Ref SecurityGroup + IamInstanceProfile: !Ref ECSInstanceProfile + UserData: + "Fn::Base64": !Sub | + #!/bin/bash + yum install -y aws-cfn-bootstrap hibagent rsync awscli + yum update -y + service amazon-ssm-agent restart + + # determine if we have an NVMe SSD attached + find /dev/nvme1 + if [ $? -eq 0 ] + then + mount_point=/var/lib/docker + + # copy existing files from mount point + service docker stop + echo 'DOCKER_STORAGE_OPTIONS="--storage-driver overlay2"' > /etc/sysconfig/docker-storage + mkdir -p /tmp$mount_point + rsync -val $mount_point/ /tmp/$mount_point/ + + # make a new filesystem and mount it + mkfs -t ext4 /dev/nvme1n1 + mkdir -p $mount_point + mount -t ext4 -o noatime /dev/nvme1n1 $mount_point + + # Copy files back to new mount point + rsync -val /tmp/$mount_point/ $mount_point/ + rm -rf /tmp$mount_point + service docker start + + # Make raid appear on reboot + echo >> /etc/fstab + echo "/dev/nvme1n1 $mount_point ext4 noatime 0 0" | tee -a /etc/fstab + fi + + /opt/aws/bin/cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource ECSLaunchConfiguration + /opt/aws/bin/cfn-signal -e $? --region ${AWS::Region} --stack ${AWS::StackName} --resource ECSAutoScalingGroup + /usr/bin/enable-ec2-spot-hibernation + echo "ECS_ENGINE_TASK_CLEANUP_WAIT_DURATION=5m" >> /etc/ecs/ecs.config + + reboot + + Metadata: + AWS::CloudFormation::Init: + config: + packages: + yum: + awslogs: [] + + commands: + 01_add_instance_to_cluster: + command: !Sub echo ECS_CLUSTER=${Cluster} >> /etc/ecs/ecs.config + files: + "/etc/cfn/cfn-hup.conf": + mode: 000400 + owner: root + group: root + content: !Sub | + [main] + stack=${AWS::StackId} + region=${AWS::Region} + + "/etc/cfn/hooks.d/cfn-auto-reloader.conf": + content: !Sub | + [cfn-auto-reloader-hook] + triggers=post.update + path=Resources.ECSLaunchConfiguration.Metadata.AWS::CloudFormation::Init + action=/opt/aws/bin/cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource ECSLaunchConfiguration + + "/etc/awslogs/awscli.conf": + content: !Sub | + [plugins] + cwlogs = cwlogs + [default] + region = ${AWS::Region} + + services: + sysvinit: + cfn-hup: + enabled: true + ensureRunning: true + files: + - /etc/cfn/cfn-hup.conf + - /etc/cfn/hooks.d/cfn-auto-reloader.conf + + # This IAM Role is attached to all of the ECS hosts. It is based on the default role + # published here: + # http://docs.aws.amazon.com/AmazonECS/latest/developerguide/instance_IAM_role.html + # + # You can add other IAM policy statements here to allow access from your ECS hosts + # to other AWS services. + + ECSRole: + Type: AWS::IAM::Role + Properties: + Path: / + RoleName: !Sub ${AWS::StackName}-ECSRole-${AWS::Region} + AssumeRolePolicyDocument: | + { + "Statement": [{ + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ec2.amazonaws.com" + } + }] + } + Policies: + - PolicyName: ecs-service + PolicyDocument: | + { + "Statement": [{ + "Effect": "Allow", + "Action": [ + "ecs:CreateCluster", + "ecs:DeregisterContainerInstance", + "ecs:DiscoverPollEndpoint", + "ecs:Poll", + "ecs:RegisterContainerInstance", + "ecs:StartTelemetrySession", + "ecs:Submit*", + "logs:CreateLogStream", + "logs:PutLogEvents", + "ecr:BatchCheckLayerAvailability", + "ecr:BatchGetImage", + "ecr:GetDownloadUrlForLayer", + "ecr:GetAuthorizationToken", + "ssm:DescribeAssociation", + "ssm:GetDeployablePatchSnapshotForInstance", + "ssm:GetDocument", + "ssm:GetManifest", + "ssm:GetParameters", + "ssm:ListAssociations", + "ssm:ListInstanceAssociations", + "ssm:PutInventory", + "ssm:PutComplianceItems", + "ssm:PutConfigurePackageResult", + "ssm:PutParameter", + "ssm:UpdateAssociationStatus", + "ssm:UpdateInstanceAssociationStatus", + "ssm:UpdateInstanceInformation", + "ec2messages:AcknowledgeMessage", + "ec2messages:DeleteMessage", + "ec2messages:FailMessage", + "ec2messages:GetEndpoint", + "ec2messages:GetMessages", + "ec2messages:SendReply", + "cloudwatch:PutMetricData", + "ec2:DescribeInstanceStatus", + "ds:CreateComputer", + "ds:DescribeDirectories", + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:DescribeLogGroups", + "logs:DescribeLogStreams", + "logs:PutLogEvents", + "s3:*" + ], + "Resource": "*" + }] + } + + ECSInstanceProfile: + Type: AWS::IAM::InstanceProfile + Properties: + Path: / + Roles: + - !Ref ECSRole + + ECSServiceAutoScalingRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + Action: + - 'sts:AssumeRole' + Effect: Allow + Principal: + Service: + - application-autoscaling.amazonaws.com + Path: / + Policies: + - PolicyName: ecs-service-autoscaling + PolicyDocument: + Statement: + Effect: Allow + Action: + - application-autoscaling:* + - cloudwatch:DescribeAlarms + - cloudwatch:PutMetricAlarm + - ecs:DescribeServices + - ecs:UpdateService + Resource: "*" + + TaskExecutionRole: + Type: AWS::IAM::Role + Properties: + Path: / + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Action: sts:AssumeRole + Effect: Allow + Principal: + Service: ecs-tasks.amazonaws.com + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy + Policies: + - PolicyName: !Sub ecs-task-S3-${AWS::StackName} + PolicyDocument: + Version: 2012-10-17 + Statement: + - + Effect: Allow + Action: + - "s3:Get*" + - "s3:List*" + - "s3:Put*" + Resource: + - !GetAtt ChainBucket.Arn + - PolicyName: !Sub ecs-task-SSM-${AWS::StackName} + PolicyDocument: + Version: 2012-10-17 + Statement: + - + Effect: Allow + Action: + - "ssm:DescribeParameters" + - "ssm:PutParameter" + - "ssm:GetParameters" + Resource: + - !Sub "arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter/${AWS::StackName}/*" + + LogGroup: + Type: AWS::Logs::LogGroup + Properties: + LogGroupName: !Sub /ecs/${AWS::StackName} + RetentionInDays: 14 + + ECSService: + Type: AWS::ECS::Service + Properties: + Cluster: !Ref Cluster + DesiredCount: !Ref DesiredCount + TaskDefinition: !Ref TaskDefinition + LaunchType: EC2 + DeploymentConfiguration: + MaximumPercent: 100 + MinimumHealthyPercent: 0 + + TaskDefinition: + Type: AWS::ECS::TaskDefinition + Properties: + Family: !Sub ${AWS::StackName}-${TaskName} + RequiresCompatibilities: + - EC2 + NetworkMode: host + ExecutionRoleArn: !Ref TaskExecutionRole + ContainerDefinitions: + - Name: !Ref TaskName + Image: !Sub ${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${Repository} + Essential: true + MemoryReservation: 6144 + Environment: + - Name: "network" + Value: !Ref Network + - Name: "syncmode" + Value: !Ref SyncMode + - Name: "connections" + Value: !Ref Connections + - Name: "netport" + Value: !Ref NetPort + - Name: "region" + Value: !Ref AWS::Region + - Name: "chainbucket" + Value: !Ref ChainBucket + - Name: "s3key" + Value: node + PortMappings: + - ContainerPort: !Ref NetPort + LogConfiguration: + LogDriver: awslogs + Options: + awslogs-region: !Ref AWS::Region + awslogs-group: !Ref LogGroup + awslogs-stream-prefix: !Ref AWS::StackName + +# CodePipeline Resources + + Repository: + Type: AWS::ECR::Repository + + CodeBuildServiceRole: + Type: AWS::IAM::Role + Properties: + Path: / + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: codebuild.amazonaws.com + Action: sts:AssumeRole + Policies: + - PolicyName: root + PolicyDocument: + Version: 2012-10-17 + Statement: + - Resource: "*" + Effect: Allow + Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents + - ecr:GetAuthorizationToken + - Resource: !Sub arn:aws:s3:::${ArtifactBucket}/* + Effect: Allow + Action: + - s3:GetObject + - s3:PutObject + - s3:GetObjectVersion + - Resource: !Sub arn:aws:ecr:${AWS::Region}:${AWS::AccountId}:repository/${Repository} + Effect: Allow + Action: + - ecr:GetDownloadUrlForLayer + - ecr:BatchGetImage + - ecr:BatchCheckLayerAvailability + - ecr:PutImage + - ecr:InitiateLayerUpload + - ecr:UploadLayerPart + - ecr:CompleteLayerUpload + + CodePipelineServiceRole: + Type: AWS::IAM::Role + Properties: + Path: / + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: codepipeline.amazonaws.com + Action: sts:AssumeRole + Policies: + - PolicyName: root + PolicyDocument: + Version: 2012-10-17 + Statement: + - Resource: + - !Sub arn:aws:s3:::${ArtifactBucket}/* + Effect: Allow + Action: + - s3:PutObject + - s3:GetObject + - s3:GetObjectVersion + - s3:GetBucketVersioning + - Resource: "*" + Effect: Allow + Action: + - ecs:DescribeServices + - ecs:DescribeTaskDefinition + - ecs:DescribeTasks + - ecs:ListTasks + - ecs:RegisterTaskDefinition + - ecs:UpdateService + - codebuild:StartBuild + - codebuild:BatchGetBuilds + - iam:PassRole + + ArtifactBucket: + Type: AWS::S3::Bucket + + CodeBuildProject: + Type: AWS::CodeBuild::Project + Properties: + Artifacts: + Type: CODEPIPELINE + Source: + Type: CODEPIPELINE + BuildSpec: | + version: 0.2 + phases: + install: + runtime-versions: + docker: 19 + pre_build: + commands: + - $(aws ecr get-login --no-include-email) + - TAG="$(echo $CODEBUILD_RESOLVED_SOURCE_VERSION | head -c 8)" + - IMAGE_URI="${REPOSITORY_URI}:${TAG}" + - cp infra/Dockerfile.updater ./Dockerfile + build: + commands: + - docker build --tag "$IMAGE_URI" . + - docker build --tag "${REPOSITORY_URI}:latest" . + post_build: + commands: + - docker push "$IMAGE_URI" + - docker push "${REPOSITORY_URI}:latest" + - printf '[{"name":"mev-geth-updater","imageUri":"%s"}]' "$IMAGE_URI" > images.json + artifacts: + files: images.json + Environment: + ComputeType: BUILD_GENERAL1_LARGE + Image: aws/codebuild/amazonlinux2-aarch64-standard:1.0 + Type: ARM_CONTAINER + PrivilegedMode: true + EnvironmentVariables: + - Name: AWS_DEFAULT_REGION + Value: !Ref AWS::Region + - Name: REPOSITORY_URI + Value: !Sub ${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${Repository} + Name: !Ref AWS::StackName + ServiceRole: !Ref CodeBuildServiceRole + + Pipeline: + Type: AWS::CodePipeline::Pipeline + Properties: + RoleArn: !GetAtt CodePipelineServiceRole.Arn + ArtifactStore: + Type: S3 + Location: !Ref ArtifactBucket + Stages: + - Name: Source + Actions: + - Name: App + ActionTypeId: + Category: Source + Owner: ThirdParty + Version: 1 + Provider: GitHub + Configuration: + Owner: !Ref GitHubUser + Repo: !Ref GitHubRepo + Branch: !Ref GitHubBranch + OAuthToken: !Ref GitHubToken + OutputArtifacts: + - Name: App + RunOrder: 1 + - Name: Build + Actions: + - Name: Build + ActionTypeId: + Category: Build + Owner: AWS + Version: 1 + Provider: CodeBuild + Configuration: + ProjectName: !Ref CodeBuildProject + InputArtifacts: + - Name: App + OutputArtifacts: + - Name: BuildOutput + RunOrder: 1 + - Name: Deploy + Actions: + - Name: Deploy + ActionTypeId: + Category: Deploy + Owner: AWS + Version: 1 + Provider: ECS + Configuration: + ClusterName: !Ref Cluster + ServiceName: !Ref ECSService + FileName: images.json + InputArtifacts: + - Name: BuildOutput + RunOrder: 1 + +Outputs: + + ClusterName: + Value: !Ref Cluster + Service: + Value: !Ref ECSService + PipelineUrl: + Value: !Sub https://console.aws.amazon.com/codepipeline/home?region=${AWS::Region}#/view/${Pipeline} \ No newline at end of file diff --git a/infra/mev-geth-updater-x86-64.yaml b/infra/mev-geth-updater-x86-64.yaml new file mode 100644 index 000000000000..a69d1bb10d18 --- /dev/null +++ b/infra/mev-geth-updater-x86-64.yaml @@ -0,0 +1,737 @@ +--- +AWSTemplateFormatVersion: 2010-09-09 + +Description: > + This template creates an automated continuous deployment pipeline to Amazon Elastic Container Service (ECS) + Created by Luke Youngblood, luke@blockscale.net + +Parameters: + # GitHub Parameters + + GitHubUser: + Type: String + Default: lyoungblood + Description: Your team or username on GitHub. + + GitHubRepo: + Type: String + Default: mev-geth + Description: The repo name of the baker service. + + GitHubBranch: + Type: String + Default: master + Description: The branch of the repo to continuously deploy. + + GitHubToken: + Type: String + NoEcho: true + Description: > + Token for the team or user specified above. (https://github.com/settings/tokens) + + # VPC Parameters + + VPC: + Type: AWS::EC2::VPC::Id + + Subnets: + Type: List + + VpcCIDR: + Type: String + Default: 172.31.0.0/16 + + # ECS Parameters + + InstanceType: + Type: String + Default: i3en.large + + KeyPair: + Type: AWS::EC2::KeyPair::KeyName + + ClusterSize: + Type: Number + Default: 1 + + DesiredCount: + Type: Number + Default: 0 + + TaskName: + Type: String + Default: mev-geth-updater + + ECSAMI: + Type: AWS::SSM::Parameter::Value + Default: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id + + # Mev-Geth Parameters + + Network: + Type: String + Default: mainnet + AllowedValues: + - mainnet + - goerli + + SyncMode: + Type: String + Default: fast + AllowedValues: + - full + - fast + - light + + Connections: + Type: Number + Default: 50 + + NetPort: + Type: Number + Default: 30303 + +Metadata: + AWS::CloudFormation::Interface: + ParameterLabels: + GitHubUser: + default: "User" + GitHubRepo: + default: "Mev-Geth GitHub Repository" + GitHubBranch: + default: "Branch in GitHub repository" + GitHubToken: + default: "Personal Access Token" + VPC: + default: "Choose which VPC the autoscaling group should be deployed to" + Subnets: + default: "Choose which subnets the autoscaling group should be deployed to" + VpcCIDR: + default: "VPC CIDR Block" + InstanceType: + default: "Which instance type should we use to build the ECS cluster?" + KeyPair: + default: "Which keypair should be used for access to the ECS cluster?" + ClusterSize: + default: "How many ECS hosts do you want to initially deploy?" + DesiredCount: + default: "How many Updater tasks do you want to initially execute?" + TaskName: + default: "The name of the Updater ECS Task" + ECSAMI: + default: "The ECS AMI ID populated from SSM." + Network: + default: "The network the Mev-Geth node should join" + SyncMode: + default: "The synchronization mode that Mev-Geth should use (full, fast, or light)" + Connections: + default: "The number of connections the Mev-Geth node should be configured with" + NetPort: + default: "The TCP/UDP port used for Mev-Geth connectivity to other Ethereum peer nodes" + ParameterGroups: + - Label: + default: GitHub Configuration + Parameters: + - GitHubRepo + - GitHubBranch + - GitHubUser + - GitHubToken + - Label: + default: VPC Configuration + Parameters: + - VPC + - Subnets + - VpcCIDR + - Label: + default: ECS Configuration + Parameters: + - InstanceType + - KeyPair + - ClusterSize + - DesiredCount + - TaskName + - ECSAMI + - Label: + default: Mev-Geth Configuration + Parameters: + - Network + - SyncMode + - Connections + - NetPort + +Resources: + # ECS Resources + + ChainBucket: + Type: AWS::S3::Bucket + + ChainBucketPolicy: + Type: AWS::S3::BucketPolicy + Properties: + Bucket: !Ref ChainBucket + PolicyDocument: + Statement: + - Action: + - s3:GetObject + - s3:ListBucket + Effect: Allow + Resource: + - Fn::Join: + - "" + - - "arn:aws:s3:::" + - Ref: "ChainBucket" + - "/*" + - Fn::Join: + - "" + - - "arn:aws:s3:::" + - Ref: "ChainBucket" + Principal: + AWS: "*" + + Cluster: + Type: AWS::ECS::Cluster + Properties: + ClusterName: !Ref AWS::StackName + + SecurityGroup: + Type: "AWS::EC2::SecurityGroup" + Properties: + GroupDescription: !Sub ${AWS::StackName}-sg + VpcId: !Ref VPC + Tags: + - Key: Name + Value: !Sub ${AWS::StackName}-sg + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 22 + ToPort: 22 + CidrIp: !Ref VpcCIDR + - IpProtocol: tcp + FromPort: !Ref NetPort + ToPort: !Ref NetPort + CidrIp: 0.0.0.0/0 + - IpProtocol: tcp + FromPort: !Ref NetPort + ToPort: !Ref NetPort + CidrIpv6: ::/0 + - IpProtocol: udp + FromPort: !Ref NetPort + ToPort: !Ref NetPort + CidrIp: 0.0.0.0/0 + - IpProtocol: udp + FromPort: !Ref NetPort + ToPort: !Ref NetPort + CidrIpv6: ::/0 + + ECSAutoScalingGroup: + Type: AWS::AutoScaling::AutoScalingGroup + Properties: + VPCZoneIdentifier: !Ref Subnets + LaunchConfigurationName: !Ref ECSLaunchConfiguration + MinSize: !Ref ClusterSize + MaxSize: !Ref ClusterSize + DesiredCapacity: !Ref ClusterSize + Tags: + - Key: Name + Value: !Sub ${AWS::StackName} ECS host + PropagateAtLaunch: true + CreationPolicy: + ResourceSignal: + Timeout: PT15M + UpdatePolicy: + AutoScalingRollingUpdate: + MinInstancesInService: 0 + MaxBatchSize: 1 + PauseTime: PT15M + SuspendProcesses: + - HealthCheck + - ReplaceUnhealthy + - AZRebalance + - AlarmNotification + - ScheduledActions + WaitOnResourceSignals: true + + ECSLaunchConfiguration: + Type: AWS::AutoScaling::LaunchConfiguration + Properties: + ImageId: !Ref ECSAMI + InstanceType: !Ref InstanceType + KeyName: !Ref KeyPair + SecurityGroups: + - !Ref SecurityGroup + IamInstanceProfile: !Ref ECSInstanceProfile + UserData: + "Fn::Base64": !Sub | + #!/bin/bash + yum install -y aws-cfn-bootstrap hibagent rsync awscli + yum update -y + service amazon-ssm-agent restart + + # determine if we have an NVMe SSD attached + find /dev/nvme1 + if [ $? -eq 0 ] + then + mount_point=/var/lib/docker + + # copy existing files from mount point + service docker stop + echo 'DOCKER_STORAGE_OPTIONS="--storage-driver overlay2"' > /etc/sysconfig/docker-storage + mkdir -p /tmp$mount_point + rsync -val $mount_point/ /tmp/$mount_point/ + + # make a new filesystem and mount it + mkfs -t ext4 /dev/nvme1n1 + mkdir -p $mount_point + mount -t ext4 -o noatime /dev/nvme1n1 $mount_point + + # Copy files back to new mount point + rsync -val /tmp/$mount_point/ $mount_point/ + rm -rf /tmp$mount_point + service docker start + + # Make raid appear on reboot + echo >> /etc/fstab + echo "/dev/nvme1n1 $mount_point ext4 noatime 0 0" | tee -a /etc/fstab + fi + + /opt/aws/bin/cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource ECSLaunchConfiguration + /opt/aws/bin/cfn-signal -e $? --region ${AWS::Region} --stack ${AWS::StackName} --resource ECSAutoScalingGroup + /usr/bin/enable-ec2-spot-hibernation + echo "ECS_ENGINE_TASK_CLEANUP_WAIT_DURATION=5m" >> /etc/ecs/ecs.config + + reboot + + Metadata: + AWS::CloudFormation::Init: + config: + packages: + yum: + awslogs: [] + + commands: + 01_add_instance_to_cluster: + command: !Sub echo ECS_CLUSTER=${Cluster} >> /etc/ecs/ecs.config + files: + "/etc/cfn/cfn-hup.conf": + mode: 000400 + owner: root + group: root + content: !Sub | + [main] + stack=${AWS::StackId} + region=${AWS::Region} + + "/etc/cfn/hooks.d/cfn-auto-reloader.conf": + content: !Sub | + [cfn-auto-reloader-hook] + triggers=post.update + path=Resources.ECSLaunchConfiguration.Metadata.AWS::CloudFormation::Init + action=/opt/aws/bin/cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource ECSLaunchConfiguration + + "/etc/awslogs/awscli.conf": + content: !Sub | + [plugins] + cwlogs = cwlogs + [default] + region = ${AWS::Region} + + services: + sysvinit: + cfn-hup: + enabled: true + ensureRunning: true + files: + - /etc/cfn/cfn-hup.conf + - /etc/cfn/hooks.d/cfn-auto-reloader.conf + + # This IAM Role is attached to all of the ECS hosts. It is based on the default role + # published here: + # http://docs.aws.amazon.com/AmazonECS/latest/developerguide/instance_IAM_role.html + # + # You can add other IAM policy statements here to allow access from your ECS hosts + # to other AWS services. + + ECSRole: + Type: AWS::IAM::Role + Properties: + Path: / + RoleName: !Sub ${AWS::StackName}-ECSRole-${AWS::Region} + AssumeRolePolicyDocument: | + { + "Statement": [{ + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ec2.amazonaws.com" + } + }] + } + Policies: + - PolicyName: ecs-service + PolicyDocument: | + { + "Statement": [{ + "Effect": "Allow", + "Action": [ + "ecs:CreateCluster", + "ecs:DeregisterContainerInstance", + "ecs:DiscoverPollEndpoint", + "ecs:Poll", + "ecs:RegisterContainerInstance", + "ecs:StartTelemetrySession", + "ecs:Submit*", + "logs:CreateLogStream", + "logs:PutLogEvents", + "ecr:BatchCheckLayerAvailability", + "ecr:BatchGetImage", + "ecr:GetDownloadUrlForLayer", + "ecr:GetAuthorizationToken", + "ssm:DescribeAssociation", + "ssm:GetDeployablePatchSnapshotForInstance", + "ssm:GetDocument", + "ssm:GetManifest", + "ssm:GetParameters", + "ssm:ListAssociations", + "ssm:ListInstanceAssociations", + "ssm:PutInventory", + "ssm:PutComplianceItems", + "ssm:PutConfigurePackageResult", + "ssm:PutParameter", + "ssm:UpdateAssociationStatus", + "ssm:UpdateInstanceAssociationStatus", + "ssm:UpdateInstanceInformation", + "ec2messages:AcknowledgeMessage", + "ec2messages:DeleteMessage", + "ec2messages:FailMessage", + "ec2messages:GetEndpoint", + "ec2messages:GetMessages", + "ec2messages:SendReply", + "cloudwatch:PutMetricData", + "ec2:DescribeInstanceStatus", + "ds:CreateComputer", + "ds:DescribeDirectories", + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:DescribeLogGroups", + "logs:DescribeLogStreams", + "logs:PutLogEvents", + "s3:*" + ], + "Resource": "*" + }] + } + + ECSInstanceProfile: + Type: AWS::IAM::InstanceProfile + Properties: + Path: / + Roles: + - !Ref ECSRole + + ECSServiceAutoScalingRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + Action: + - "sts:AssumeRole" + Effect: Allow + Principal: + Service: + - application-autoscaling.amazonaws.com + Path: / + Policies: + - PolicyName: ecs-service-autoscaling + PolicyDocument: + Statement: + Effect: Allow + Action: + - application-autoscaling:* + - cloudwatch:DescribeAlarms + - cloudwatch:PutMetricAlarm + - ecs:DescribeServices + - ecs:UpdateService + Resource: "*" + + TaskExecutionRole: + Type: AWS::IAM::Role + Properties: + Path: / + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Action: sts:AssumeRole + Effect: Allow + Principal: + Service: ecs-tasks.amazonaws.com + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy + Policies: + - PolicyName: !Sub ecs-task-S3-${AWS::StackName} + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - "s3:Get*" + - "s3:List*" + - "s3:Put*" + Resource: + - !GetAtt ChainBucket.Arn + - PolicyName: !Sub ecs-task-SSM-${AWS::StackName} + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - "ssm:DescribeParameters" + - "ssm:PutParameter" + - "ssm:GetParameters" + Resource: + - !Sub "arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter/${AWS::StackName}/*" + + LogGroup: + Type: AWS::Logs::LogGroup + Properties: + LogGroupName: !Sub /ecs/${AWS::StackName} + RetentionInDays: 14 + + ECSService: + Type: AWS::ECS::Service + Properties: + Cluster: !Ref Cluster + DesiredCount: !Ref DesiredCount + TaskDefinition: !Ref TaskDefinition + LaunchType: EC2 + DeploymentConfiguration: + MaximumPercent: 100 + MinimumHealthyPercent: 0 + + TaskDefinition: + Type: AWS::ECS::TaskDefinition + Properties: + Family: !Sub ${AWS::StackName}-${TaskName} + RequiresCompatibilities: + - EC2 + NetworkMode: host + ExecutionRoleArn: !Ref TaskExecutionRole + ContainerDefinitions: + - Name: !Ref TaskName + Image: !Sub ${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${Repository} + Essential: true + MemoryReservation: 6144 + Environment: + - Name: "network" + Value: !Ref Network + - Name: "syncmode" + Value: !Ref SyncMode + - Name: "connections" + Value: !Ref Connections + - Name: "netport" + Value: !Ref NetPort + - Name: "region" + Value: !Ref AWS::Region + - Name: "chainbucket" + Value: !Ref ChainBucket + - Name: "s3key" + Value: node + PortMappings: + - ContainerPort: !Ref NetPort + LogConfiguration: + LogDriver: awslogs + Options: + awslogs-region: !Ref AWS::Region + awslogs-group: !Ref LogGroup + awslogs-stream-prefix: !Ref AWS::StackName + + # CodePipeline Resources + + Repository: + Type: AWS::ECR::Repository + + CodeBuildServiceRole: + Type: AWS::IAM::Role + Properties: + Path: / + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: codebuild.amazonaws.com + Action: sts:AssumeRole + Policies: + - PolicyName: root + PolicyDocument: + Version: 2012-10-17 + Statement: + - Resource: "*" + Effect: Allow + Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents + - ecr:GetAuthorizationToken + - Resource: !Sub arn:aws:s3:::${ArtifactBucket}/* + Effect: Allow + Action: + - s3:GetObject + - s3:PutObject + - s3:GetObjectVersion + - Resource: !Sub arn:aws:ecr:${AWS::Region}:${AWS::AccountId}:repository/${Repository} + Effect: Allow + Action: + - ecr:GetDownloadUrlForLayer + - ecr:BatchGetImage + - ecr:BatchCheckLayerAvailability + - ecr:PutImage + - ecr:InitiateLayerUpload + - ecr:UploadLayerPart + - ecr:CompleteLayerUpload + + CodePipelineServiceRole: + Type: AWS::IAM::Role + Properties: + Path: / + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: codepipeline.amazonaws.com + Action: sts:AssumeRole + Policies: + - PolicyName: root + PolicyDocument: + Version: 2012-10-17 + Statement: + - Resource: + - !Sub arn:aws:s3:::${ArtifactBucket}/* + Effect: Allow + Action: + - s3:PutObject + - s3:GetObject + - s3:GetObjectVersion + - s3:GetBucketVersioning + - Resource: "*" + Effect: Allow + Action: + - ecs:DescribeServices + - ecs:DescribeTaskDefinition + - ecs:DescribeTasks + - ecs:ListTasks + - ecs:RegisterTaskDefinition + - ecs:UpdateService + - codebuild:StartBuild + - codebuild:BatchGetBuilds + - iam:PassRole + + ArtifactBucket: + Type: AWS::S3::Bucket + + CodeBuildProject: + Type: AWS::CodeBuild::Project + Properties: + Artifacts: + Type: CODEPIPELINE + Source: + Type: CODEPIPELINE + BuildSpec: | + version: 0.2 + phases: + install: + runtime-versions: + docker: 19 + pre_build: + commands: + - $(aws ecr get-login --no-include-email) + - TAG="$(echo $CODEBUILD_RESOLVED_SOURCE_VERSION | head -c 8)" + - IMAGE_URI="${REPOSITORY_URI}:${TAG}" + - cp infra/Dockerfile.updater ./Dockerfile + build: + commands: + - docker build --tag "$IMAGE_URI" . + - docker build --tag "${REPOSITORY_URI}:latest" . + post_build: + commands: + - docker push "$IMAGE_URI" + - docker push "${REPOSITORY_URI}:latest" + - printf '[{"name":"mev-geth-updater","imageUri":"%s"}]' "$IMAGE_URI" > images.json + artifacts: + files: images.json + Environment: + ComputeType: BUILD_GENERAL1_SMALL + Image: aws/codebuild/docker:17.09.0 + Type: LINUX_CONTAINER + PrivilegedMode: true + EnvironmentVariables: + - Name: AWS_DEFAULT_REGION + Value: !Ref AWS::Region + - Name: REPOSITORY_URI + Value: !Sub ${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${Repository} + Name: !Ref AWS::StackName + ServiceRole: !Ref CodeBuildServiceRole + + Pipeline: + Type: AWS::CodePipeline::Pipeline + Properties: + RoleArn: !GetAtt CodePipelineServiceRole.Arn + ArtifactStore: + Type: S3 + Location: !Ref ArtifactBucket + Stages: + - Name: Source + Actions: + - Name: App + ActionTypeId: + Category: Source + Owner: ThirdParty + Version: 1 + Provider: GitHub + Configuration: + Owner: !Ref GitHubUser + Repo: !Ref GitHubRepo + Branch: !Ref GitHubBranch + OAuthToken: !Ref GitHubToken + OutputArtifacts: + - Name: App + RunOrder: 1 + - Name: Build + Actions: + - Name: Build + ActionTypeId: + Category: Build + Owner: AWS + Version: 1 + Provider: CodeBuild + Configuration: + ProjectName: !Ref CodeBuildProject + InputArtifacts: + - Name: App + OutputArtifacts: + - Name: BuildOutput + RunOrder: 1 + - Name: Deploy + Actions: + - Name: Deploy + ActionTypeId: + Category: Deploy + Owner: AWS + Version: 1 + Provider: ECS + Configuration: + ClusterName: !Ref Cluster + ServiceName: !Ref ECSService + FileName: images.json + InputArtifacts: + - Name: BuildOutput + RunOrder: 1 + +Outputs: + ClusterName: + Value: !Ref Cluster + Service: + Value: !Ref ECSService + PipelineUrl: + Value: !Sub https://console.aws.amazon.com/codepipeline/home?region=${AWS::Region}#/view/${Pipeline} diff --git a/infra/start-mev-geth-node.sh b/infra/start-mev-geth-node.sh new file mode 100755 index 000000000000..05ad50c61003 --- /dev/null +++ b/infra/start-mev-geth-node.sh @@ -0,0 +1,96 @@ +#!/bin/sh -x +# Starts the Mev-Geth node client +# Written by Luke Youngblood, luke@blockscale.net + +# network=mainnet # normally set by environment +# syncmode=fast # normally set by environment +# rpcport=8545 # normally set by environment +# wsport=8546 # normally set by environment +# netport=30303 # normally set by environment + +init_node() { + # You can put any commands you would like to run to initialize the node here. + echo Initializing node... +} + +start_node() { + if [ $network = "goerli" ] + then + geth \ + --port $netport \ + --http \ + --http.addr 0.0.0.0 \ + --http.port $rpcport \ + --http.api eth,net,web3 \ + --http.vhosts '*' \ + --http.corsdomain '*' \ + --graphql \ + --graphql.corsdomain '*' \ + --graphql.vhosts '*' \ + --ws \ + --ws.addr 0.0.0.0 \ + --ws.port $wsport \ + --ws.api eth,net,web3 \ + --ws.origins '*' \ + --syncmode $syncmode \ + --cache 4096 \ + --maxpeers $connections \ + --goerli + if [ $? -ne 0 ] + then + echo "Node failed to start; exiting." + exit 1 + fi + else + geth \ + --port $netport \ + --http \ + --http.addr 0.0.0.0 \ + --http.port $rpcport \ + --http.api eth,net,web3 \ + --http.vhosts '*' \ + --http.corsdomain '*' \ + --graphql \ + --graphql.corsdomain '*' \ + --graphql.vhosts '*' \ + --ws \ + --ws.addr 0.0.0.0 \ + --ws.port $wsport \ + --ws.api eth,net,web3 \ + --ws.origins '*' \ + --syncmode $syncmode \ + --cache 4096 \ + --maxpeers $connections + if [ $? -ne 0 ] + then + echo "Node failed to start; exiting." + exit 1 + fi + fi +} + +s3_sync() { + # Determine data directory + if [ $network = "goerli" ] + then + datadir=/root/.ethereum/goerli/geth/chaindata + else + datadir=/root/.ethereum/geth/chaindata + fi + # If the current1 key exists, node1 is the most current set of blockchain data + echo "A 404 error below is expected and nothing to be concerned with." + aws s3api head-object --request-payer requester --bucket $chainbucket --key current1 + if [ $? -eq 0 ] + then + s3key=node1 + else + s3key=node2 + fi + aws s3 sync --only-show-errors --request-payer requester --region $region s3://$chainbucket/$s3key $datadir +} + +# main + +init_node +s3_sync +start_node diff --git a/infra/start-mev-geth-updater.sh b/infra/start-mev-geth-updater.sh new file mode 100755 index 000000000000..11a6a533aa14 --- /dev/null +++ b/infra/start-mev-geth-updater.sh @@ -0,0 +1,181 @@ +#!/bin/sh -x +# Starts the Mev-Geth updater client +# Written by Luke Youngblood, luke@blockscale.net + +# netport=30303 # normally set by environment + +init_node() { + # Initialization steps can go here + echo Initializing node... + aws configure set default.s3.max_concurrent_requests 64 + aws configure set default.s3.max_queue_size 20000 +} + +start_node() { + if [ $network = "goerli" ] + then + geth \ + --port $netport \ + --syncmode $syncmode \ + --cache 4096 \ + --maxpeers $connections \ + --goerli & + if [ $? -ne 0 ] + then + echo "Node failed to start; exiting." + exit 1 + fi + else + geth \ + --port $netport \ + --syncmode $syncmode \ + --cache 4096 \ + --maxpeers $connections & + if [ $? -ne 0 ] + then + echo "Node failed to start; exiting." + exit 1 + fi + fi +} + +s3_sync_down() { + # Determine data directory + if [ $network = "goerli" ] + then + datadir=/root/.ethereum/goerli/geth/chaindata + else + datadir=/root/.ethereum/geth/chaindata + fi + + # If the current1 object exists, node1 is the key we should download + echo "A 404 error below is expected and nothing to be concerned with." + aws s3api head-object --bucket $chainbucket --key current1 + if [ $? -eq 0 ] + then + echo "current1 key exists; downloading node1" + s3key=node1 + else + echo "current1 key doesn't exist; downloading node2" + s3key=node2 + fi + + aws s3 sync --region $region --only-show-errors s3://$chainbucket/$s3key $datadir + if [ $? -ne 0 ] + then + echo "aws s3 sync command failed; exiting." + exit 2 + fi +} + +kill_node() { + tries=0 + while [ ! -z `ps -ef |grep geth|grep -v geth-updater|grep -v grep|awk '{print $1}'` ] + do + ps -ef |grep geth|grep -v geth-updater|grep -v grep + pid=`ps -ef |grep geth|grep -v geth-updater|grep -v grep|awk '{print $1}'` + kill $pid + sleep 30 + echo "Waiting for the node to shutdown cleanly... try number $tries" + let "tries+=1" + if [ $tries -gt 29 ] + then + echo "Node has not stopped cleanly after $tries, forcibly killing." + ps -ef |grep geth|grep -v geth-updater|grep -v grep + pid=`ps -ef |grep geth|grep -v geth-updater|grep -v grep|awk '{print $1}'` + kill -9 $pid + fi + if [ $tries -gt 30 ] + then + echo "Node has not stopped cleanly after $tries, exiting..." + exit 3 + fi + done +} + +s3_sync_up() { + # Determine data directory + if [ $network = "goerli" ] + then + datadir=/root/.ethereum/goerli/geth/chaindata + else + datadir=/root/.ethereum/geth/chaindata + fi + + # If the current1 object exists, node1 is the folder that clients will download, so we should update node2 + aws s3api head-object --bucket $chainbucket --key current1 + if [ $? -eq 0 ] + then + echo "current1 key exists; updating node2" + s3key=node2 + else + echo "current1 key doesn't exist; updating node1" + s3key=node1 + fi + + aws s3 sync --delete --region $region --only-show-errors --acl public-read $datadir s3://$chainbucket/$s3key + if [ $? -ne 0 ] + then + echo "aws s3 sync upload command failed; exiting." + exit 4 + fi + + if [ "$s3key" = "node2" ] + then + echo "Removing current1 key, as the node2 key was just updated." + aws s3 rm --region $region s3://$chainbucket/current1 + if [ $? -ne 0 ] + then + echo "aws s3 rm command failed; retrying." + sleep 5 + aws s3 rm --region $region s3://$chainbucket/current1 + if [ $? -ne 0 ] + then + echo "aws s3 rm command failed; exiting." + exit 5 + fi + fi + else + echo "Touching current1 key, as the node1 key was just updated." + touch ~/current1 + aws s3 cp --region $region --acl public-read ~/current1 s3://$chainbucket/ + if [ $? -ne 0 ] + then + echo "aws s3 cp command failed; retrying." + sleep 5 + aws s3 cp --region $region --acl public-read ~/current1 s3://$chainbucket/ + if [ $? -ne 0 ] + then + echo "aws s3 cp command failed; exiting." + exit 6 + fi + fi + fi +} + +continuous() { + # This function continuously stops the node every hour + # and syncs the chain data with S3, then restarts the node. + while true + do + echo "Sleeping for 60 minutes at `date`..." + sleep 3600 + echo "Cleanly shutting down the node so we can update S3 with the latest chaindata at `date`..." + kill_node + echo "Syncing chain data to S3 at `date`..." + s3_sync_up + echo "Restarting the node after syncing to S3 at `date`..." + start_node + done +} + +# main + +echo "Initializing the node at `date`..." +init_node +echo "Syncing initial chain data with stored chain data in S3 at `date`..." +s3_sync_down +echo "Starting the node at `date`..." +start_node +echo "Starting the continuous loop at `date`..." +continuous From d5e0557b5e24b943769be62e47501f3df1581272 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Mon, 29 Mar 2021 20:28:58 -0700 Subject: [PATCH 05/17] Fix flashbots miners not correctly handling reorgs --- miner/worker.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/miner/worker.go b/miner/worker.go index 784a1d7ae443..d9ed682fbf00 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -585,8 +585,8 @@ func (w *worker) taskLoop() { stopCh chan struct{} prev common.Hash - prevNumber *big.Int - prevProfit *big.Int + prevParentHash common.Hash + prevProfit *big.Int ) // interrupt aborts the in-flight sealing task. @@ -608,17 +608,19 @@ func (w *worker) taskLoop() { continue } + taskParentHash := task.block.Header().ParentHash // reject new tasks which don't profit - if prevNumber != nil && prevProfit != nil && - task.block.Number().Cmp(prevNumber) == 0 && task.profit.Cmp(prevProfit) < 0 { + if taskParentHash == prevParentHash && + prevProfit != nil && task.profit.Cmp(prevProfit) < 0 { continue } - prevNumber, prevProfit = task.block.Number(), task.profit + prevParentHash = taskParentHash + prevProfit = task.profit // Interrupt previous sealing operation interrupt() stopCh, prev = make(chan struct{}), sealHash - log.Info("Proposed miner block", "blockNumber", prevNumber, "profit", prevProfit, "isFlashbots", task.isFlashbots, "sealhash", sealHash) + log.Info("Proposed miner block", "blockNumber", task.block.Number(), "profit", prevProfit, "isFlashbots", task.isFlashbots, "sealhash", sealHash, "parentHash", prevParentHash) if w.skipSealHook != nil && w.skipSealHook(task) { continue } From cf258d634d0c74fc489de4049ae4d0a401b632dc Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Wed, 31 Mar 2021 14:36:18 -0700 Subject: [PATCH 06/17] Change flashbots bundle pricing formula to ignore gas fees --- miner/worker.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/miner/worker.go b/miner/worker.go index d9ed682fbf00..bf9de255036c 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -1277,6 +1277,7 @@ func (w *worker) computeBundleGas(bundle types.Transactions, parent *types.Block var totalGasUsed uint64 = 0 var tempGasUsed uint64 + gasFees := new(big.Int) coinbaseBalanceBefore := env.state.GetBalance(w.coinbase) @@ -1286,13 +1287,12 @@ func (w *worker) computeBundleGas(bundle types.Transactions, parent *types.Block return nil, 0, err } totalGasUsed += receipt.GasUsed + gasFees.Add(gasFees, new(big.Int).Mul(big.NewInt(int64(totalGasUsed)), tx.GasPrice())) } coinbaseBalanceAfter := env.state.GetBalance(w.coinbase) - coinbaseDiff := new(big.Int).Sub(coinbaseBalanceAfter, coinbaseBalanceBefore) - totalEth := new(big.Int) - totalEth.Add(totalEth, coinbaseDiff) + coinbaseDiff := new(big.Int).Sub(new(big.Int).Sub(coinbaseBalanceAfter, gasFees), coinbaseBalanceBefore) - return totalEth, totalGasUsed, nil + return coinbaseDiff, totalGasUsed, nil } // copyReceipts makes a deep copy of the given receipts. From f6215858e67c0b02af0fefe2474202812721ec63 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Sat, 10 Apr 2021 14:45:13 -0700 Subject: [PATCH 07/17] Discard bundles with reverting txs Fixes #30 --- miner/worker.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/miner/worker.go b/miner/worker.go index bf9de255036c..9ad092f22a2b 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -1286,6 +1286,10 @@ func (w *worker) computeBundleGas(bundle types.Transactions, parent *types.Block if err != nil { return nil, 0, err } + if receipt.Status == types.ReceiptStatusFailed { + return nil, 0, errors.New("revert") + } + totalGasUsed += receipt.GasUsed gasFees.Add(gasFees, new(big.Int).Mul(big.NewInt(int64(totalGasUsed)), tx.GasPrice())) } From 82b60ce0f0342179f61e6db6853a026247d73925 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Tue, 6 Apr 2021 16:03:03 -0700 Subject: [PATCH 08/17] Change pricing formula to ignore gas from txs in the txpool Fixes #39 --- miner/worker.go | 117 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 81 insertions(+), 36 deletions(-) diff --git a/miner/worker.go b/miner/worker.go index 9ad092f22a2b..8f8f1731dc57 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -620,7 +620,7 @@ func (w *worker) taskLoop() { // Interrupt previous sealing operation interrupt() stopCh, prev = make(chan struct{}), sealHash - log.Info("Proposed miner block", "blockNumber", task.block.Number(), "profit", prevProfit, "isFlashbots", task.isFlashbots, "sealhash", sealHash, "parentHash", prevParentHash) + log.Info("Proposed miner block", "blockNumber", task.block.Number(), "profit", ethIntToFloat(prevProfit), "isFlashbots", task.isFlashbots, "sealhash", sealHash, "parentHash", prevParentHash) if w.skipSealHook != nil && w.skipSealHook(task) { continue } @@ -1185,9 +1185,16 @@ func (w *worker) commitNewWork(interrupt *int32, noempty bool, timestamp int64) log.Error("Failed to fetch pending transactions", "err", err) return } - maxBundle, bundlePrice, ethToCoinbase, gasUsed := w.findMostProfitableBundle(bundles, w.coinbase, parent, header) - log.Info("Flashbots bundle", "ethToCoinbase", ethToCoinbase, "gasUsed", gasUsed, "bundlePrice", bundlePrice, "bundleLength", len(maxBundle)) - if w.commitBundle(maxBundle, w.coinbase, interrupt) { + bundle, err := w.findMostProfitableBundle(bundles, w.coinbase, parent, header, pending) + if err != nil { + log.Error("Failed to generate flashbots bundle", "err", err) + return + } + log.Info("Flashbots bundle", "ethToCoinbase", ethIntToFloat(bundle.totalEth), "gasUsed", bundle.totalGasUsed, "bundleScore", bundle.mevGasPrice, "bundleLength", len(bundle.txs)) + if len(bundle.txs) == 0 { + return + } + if w.commitBundle(bundle.txs, w.coinbase, interrupt) { return } } @@ -1225,7 +1232,7 @@ func (w *worker) commit(uncles []*types.Header, interval func(), update bool, st w.unconfirmed.Shift(block.NumberU64() - 1) log.Info("Commit new mining work", "number", block.Number(), "sealhash", w.engine.SealHash(block.Header()), "uncles", len(uncles), "txs", w.current.tcount, - "gas", block.GasUsed(), "fees", totalFees(block, receipts), + "gas", block.GasUsed(), "fees", totalFees(block, receipts), "profit", ethIntToFloat(w.current.profit), "elapsed", common.PrettyDuration(time.Since(start)), "isFlashbots", w.flashbots.isFlashbots) @@ -1239,64 +1246,94 @@ func (w *worker) commit(uncles []*types.Header, interval func(), update bool, st return nil } -func (w *worker) findMostProfitableBundle(bundles []types.Transactions, coinbase common.Address, parent *types.Block, header *types.Header) (types.Transactions, *big.Int, *big.Int, uint64) { - maxBundlePrice := new(big.Int) - maxTotalEth := new(big.Int) - var maxTotalGasUsed uint64 - maxBundle := types.Transactions{} +type simulatedBundle struct { + txs types.Transactions + mevGasPrice *big.Int + totalEth *big.Int + totalGasUsed uint64 +} + +func (w *worker) findMostProfitableBundle(bundles []types.Transactions, coinbase common.Address, parent *types.Block, header *types.Header, pendingTxs map[common.Address]types.Transactions) (simulatedBundle, error) { + maxBundle := simulatedBundle{mevGasPrice: new(big.Int)} + for _, bundle := range bundles { + state, err := w.chain.StateAt(parent.Root()) + if err != nil { + return simulatedBundle{}, err + } + gasPool := new(core.GasPool).AddGas(header.GasLimit) if len(bundle) == 0 { continue } - totalEth, totalGasUsed, err := w.computeBundleGas(bundle, parent, header) + simmed, err := w.computeBundleGas(bundle, parent, header, state, gasPool, pendingTxs) if err != nil { log.Debug("Error computing gas for a bundle", "error", err) continue } - mevGasPrice := new(big.Int).Div(totalEth, new(big.Int).SetUint64(totalGasUsed)) - if mevGasPrice.Cmp(maxBundlePrice) > 0 { - maxBundle = bundle - maxBundlePrice = mevGasPrice - maxTotalEth = totalEth - maxTotalGasUsed = totalGasUsed + if simmed.mevGasPrice.Cmp(maxBundle.mevGasPrice) > 0 { + maxBundle = simmed } } - return maxBundle, maxBundlePrice, maxTotalEth, maxTotalGasUsed + return maxBundle, nil } // Compute the adjusted gas price for a whole bundle // Done by calculating all gas spent, adding transfers to the coinbase, and then dividing by gas used -func (w *worker) computeBundleGas(bundle types.Transactions, parent *types.Block, header *types.Header) (*big.Int, uint64, error) { - env, err := w.generateEnv(parent, header) - if err != nil { - return nil, 0, err - } - +func (w *worker) computeBundleGas(bundle types.Transactions, parent *types.Block, header *types.Header, state *state.StateDB, gasPool *core.GasPool, pendingTxs map[common.Address]types.Transactions) (simulatedBundle, error) { var totalGasUsed uint64 = 0 var tempGasUsed uint64 - gasFees := new(big.Int) - - coinbaseBalanceBefore := env.state.GetBalance(w.coinbase) + totalEth := new(big.Int) for _, tx := range bundle { - receipt, err := core.ApplyTransaction(w.chainConfig, w.chain, &w.coinbase, env.gasPool, env.state, env.header, tx, &tempGasUsed, *w.chain.GetVMConfig()) + coinbaseBalanceBefore := state.GetBalance(w.coinbase) + + receipt, err := core.ApplyTransaction(w.chainConfig, w.chain, &w.coinbase, gasPool, state, header, tx, &tempGasUsed, *w.chain.GetVMConfig()) if err != nil { - return nil, 0, err + return simulatedBundle{}, err } if receipt.Status == types.ReceiptStatusFailed { - return nil, 0, errors.New("revert") + return simulatedBundle{}, errors.New("revert") } totalGasUsed += receipt.GasUsed - gasFees.Add(gasFees, new(big.Int).Mul(big.NewInt(int64(totalGasUsed)), tx.GasPrice())) + + from, err := types.Sender(w.current.signer, tx) + if err != nil { + return simulatedBundle{}, err + } + + txInPendingPool := false + // check if tx is in pending pool + if accountTxs, ok := pendingTxs[from]; ok { + txNonce := tx.Nonce() + + for _, accountTx := range accountTxs { + if accountTx.Nonce() == txNonce { + txInPendingPool = true + break + } + } + } + + coinbaseBalanceAfter := state.GetBalance(w.coinbase) + totalEth = totalEth.Add(totalEth, coinbaseBalanceAfter.Sub(coinbaseBalanceAfter, coinbaseBalanceBefore)) + + if txInPendingPool { + // If tx is in pending pool, ignore the gas fees + gasUsed := new(big.Int).SetUint64(receipt.GasUsed) + totalEth.Sub(totalEth, gasUsed.Mul(gasUsed, tx.GasPrice())) + } } - coinbaseBalanceAfter := env.state.GetBalance(w.coinbase) - coinbaseDiff := new(big.Int).Sub(new(big.Int).Sub(coinbaseBalanceAfter, gasFees), coinbaseBalanceBefore) - return coinbaseDiff, totalGasUsed, nil + return simulatedBundle{ + txs: bundle, + mevGasPrice: new(big.Int).Div(totalEth, new(big.Int).SetUint64(totalGasUsed)), + totalEth: totalEth, + totalGasUsed: totalGasUsed, + }, nil } // copyReceipts makes a deep copy of the given receipts. @@ -1317,12 +1354,20 @@ func (w *worker) postSideBlock(event core.ChainSideEvent) { } } -// totalFees computes total consumed miner fees in ETH. Block transactions and receipts have to have the same order. +// ethIntToFloat is for formatting a big.Int in wei to eth +func ethIntToFloat(eth *big.Int) *big.Float { + if eth == nil { + return big.NewFloat(0) + } + return new(big.Float).Quo(new(big.Float).SetInt(eth), new(big.Float).SetInt(big.NewInt(params.Ether))) +} + +// totalFees computes total consumed fees in ETH. Block transactions and receipts have to have the same order. func totalFees(block *types.Block, receipts []*types.Receipt) *big.Float { feesWei := new(big.Int) for i, tx := range block.Transactions() { minerFee, _ := tx.EffectiveGasTip(block.BaseFee()) feesWei.Add(feesWei, new(big.Int).Mul(new(big.Int).SetUint64(receipts[i].GasUsed), minerFee)) } - return new(big.Float).Quo(new(big.Float).SetInt(feesWei), new(big.Float).SetInt(big.NewInt(params.Ether))) + return ethIntToFloat(feesWei) } From 6f768de916224771941258bc2bf7277e97f8991b Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Thu, 8 Apr 2021 20:35:50 -0700 Subject: [PATCH 09/17] Use object in eth_sendBundle params and add revertingTxHashes param Fixes #40 Fixes #30 --- core/mev_bundle.go | 30 ------------------------- core/tx_pool.go | 31 +++++++++++++------------- core/types/transaction.go | 8 +++++++ eth/api_backend.go | 4 ++-- internal/ethapi/api.go | 29 ++++++++++++++++++------ internal/ethapi/backend.go | 2 +- internal/web3ext/web3ext.go | 2 +- les/api_backend.go | 4 ++-- light/txpool.go | 2 +- miner/worker.go | 44 ++++++++++++++++++++++--------------- 10 files changed, 79 insertions(+), 77 deletions(-) delete mode 100644 core/mev_bundle.go diff --git a/core/mev_bundle.go b/core/mev_bundle.go deleted file mode 100644 index 257411708e74..000000000000 --- a/core/mev_bundle.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package core - -import ( - "math/big" - - "github.com/ethereum/go-ethereum/core/types" -) - -type mevBundle struct { - txs types.Transactions - blockNumber *big.Int - minTimestamp uint64 - maxTimestamp uint64 -} diff --git a/core/tx_pool.go b/core/tx_pool.go index 592d9bddf9b9..2ad073ae3a53 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -245,7 +245,7 @@ type TxPool struct { pending map[common.Address]*txList // All currently processable transactions queue map[common.Address]*txList // Queued but non-processable transactions beats map[common.Address]time.Time // Last heartbeat from each known account - mevBundles []mevBundle + mevBundles []types.MevBundle all *txLookup // All transactions to allow lookups priced *txPricedList // All transactions sorted by price @@ -544,53 +544,54 @@ func (pool *TxPool) Pending(enforceTips bool) (map[common.Address]types.Transact } /// AllMevBundles returns all the MEV Bundles currently in the pool -func (pool *TxPool) AllMevBundles() []mevBundle { +func (pool *TxPool) AllMevBundles() []types.MevBundle { return pool.mevBundles } // MevBundles returns a list of bundles valid for the given blockNumber/blockTimestamp // also prunes bundles that are outdated -func (pool *TxPool) MevBundles(blockNumber *big.Int, blockTimestamp uint64) ([]types.Transactions, error) { +func (pool *TxPool) MevBundles(blockNumber *big.Int, blockTimestamp uint64) ([]types.MevBundle, error) { pool.mu.Lock() defer pool.mu.Unlock() // returned values - var txBundles []types.Transactions + var ret []types.MevBundle // rolled over values - var bundles []mevBundle + var bundles []types.MevBundle for _, bundle := range pool.mevBundles { // Prune outdated bundles - if (bundle.maxTimestamp != 0 && blockTimestamp > bundle.maxTimestamp) || blockNumber.Cmp(bundle.blockNumber) > 0 { + if (bundle.MaxTimestamp != 0 && blockTimestamp > bundle.MaxTimestamp) || blockNumber.Cmp(bundle.BlockNumber) > 0 { continue } // Roll over future bundles - if (bundle.minTimestamp != 0 && blockTimestamp < bundle.minTimestamp) || blockNumber.Cmp(bundle.blockNumber) < 0 { + if (bundle.MinTimestamp != 0 && blockTimestamp < bundle.MinTimestamp) || blockNumber.Cmp(bundle.BlockNumber) < 0 { bundles = append(bundles, bundle) continue } // return the ones which are in time - txBundles = append(txBundles, bundle.txs) + ret = append(ret, bundle) // keep the bundles around internally until they need to be pruned bundles = append(bundles, bundle) } pool.mevBundles = bundles - return txBundles, nil + return ret, nil } // AddMevBundle adds a mev bundle to the pool -func (pool *TxPool) AddMevBundle(txs types.Transactions, blockNumber *big.Int, minTimestamp, maxTimestamp uint64) error { +func (pool *TxPool) AddMevBundle(txs types.Transactions, blockNumber *big.Int, minTimestamp, maxTimestamp uint64, revertingTxHashes []common.Hash) error { pool.mu.Lock() defer pool.mu.Unlock() - pool.mevBundles = append(pool.mevBundles, mevBundle{ - txs: txs, - blockNumber: blockNumber, - minTimestamp: minTimestamp, - maxTimestamp: maxTimestamp, + pool.mevBundles = append(pool.mevBundles, types.MevBundle{ + Txs: txs, + BlockNumber: blockNumber, + MinTimestamp: minTimestamp, + MaxTimestamp: maxTimestamp, + RevertingTxHashes: revertingTxHashes, }) return nil } diff --git a/core/types/transaction.go b/core/types/transaction.go index a556f4b57fba..c68b03b3ee13 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -632,3 +632,11 @@ func (m Message) Nonce() uint64 { return m.nonce } func (m Message) Data() []byte { return m.data } func (m Message) AccessList() AccessList { return m.accessList } func (m Message) CheckNonce() bool { return m.checkNonce } + +type MevBundle struct { + Txs Transactions + BlockNumber *big.Int + MinTimestamp uint64 + MaxTimestamp uint64 + RevertingTxHashes []common.Hash +} diff --git a/eth/api_backend.go b/eth/api_backend.go index c380bb1394f6..321fa2ad8191 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -234,8 +234,8 @@ func (b *EthAPIBackend) SendTx(ctx context.Context, signedTx *types.Transaction) return b.eth.txPool.AddLocal(signedTx) } -func (b *EthAPIBackend) SendBundle(ctx context.Context, txs types.Transactions, blockNumber rpc.BlockNumber, minTimestamp uint64, maxTimestamp uint64) error { - return b.eth.txPool.AddMevBundle(txs, big.NewInt(blockNumber.Int64()), minTimestamp, maxTimestamp) +func (b *EthAPIBackend) SendBundle(ctx context.Context, txs types.Transactions, blockNumber rpc.BlockNumber, minTimestamp uint64, maxTimestamp uint64, revertingTxHashes []common.Hash) error { + return b.eth.txPool.AddMevBundle(txs, big.NewInt(blockNumber.Int64()), minTimestamp, maxTimestamp, revertingTxHashes) } func (b *EthAPIBackend) GetPoolTransactions() (types.Transactions, error) { diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 32b6aa1f6a78..68e69327e7c4 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -2071,12 +2071,27 @@ func NewPrivateTxBundleAPI(b Backend) *PrivateTxBundleAPI { return &PrivateTxBundleAPI{b} } +// SendBundleArgs represents the arguments for a call. +type SendBundleArgs struct { + Txs []hexutil.Bytes `json:"txs"` + BlockNumber rpc.BlockNumber `json:"blockNumber"` + MinTimestamp *uint64 `json:"minTimestamp"` + MaxTimestamp *uint64 `json:"maxTimestamp"` + RevertingTxHashes []common.Hash `json:"revertingTxHashes"` +} + // SendBundle will add the signed transaction to the transaction pool. // The sender is responsible for signing the transaction and using the correct nonce and ensuring validity -func (s *PrivateTxBundleAPI) SendBundle(ctx context.Context, encodedTxs []hexutil.Bytes, blockNumber rpc.BlockNumber, minTimestampPtr, maxTimestampPtr *uint64) error { +func (s *PrivateTxBundleAPI) SendBundle(ctx context.Context, args SendBundleArgs) error { var txs types.Transactions + if len(args.Txs) == 0 { + return errors.New("bundle missing txs") + } + if args.BlockNumber == 0 { + return errors.New("bundle missing blockNumber") + } - for _, encodedTx := range encodedTxs { + for _, encodedTx := range args.Txs { tx := new(types.Transaction) if err := rlp.DecodeBytes(encodedTx, tx); err != nil { return err @@ -2085,12 +2100,12 @@ func (s *PrivateTxBundleAPI) SendBundle(ctx context.Context, encodedTxs []hexuti } var minTimestamp, maxTimestamp uint64 - if minTimestampPtr != nil { - minTimestamp = *minTimestampPtr + if args.MinTimestamp != nil { + minTimestamp = *args.MinTimestamp } - if maxTimestampPtr != nil { - maxTimestamp = *maxTimestampPtr + if args.MaxTimestamp != nil { + maxTimestamp = *args.MaxTimestamp } - return s.b.SendBundle(ctx, txs, blockNumber, minTimestamp, maxTimestamp) + return s.b.SendBundle(ctx, txs, args.BlockNumber, minTimestamp, maxTimestamp, args.RevertingTxHashes) } diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index a31a19b70c7c..71eed5d3e9c2 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -71,7 +71,7 @@ type Backend interface { // Transaction pool API SendTx(ctx context.Context, signedTx *types.Transaction) error - SendBundle(ctx context.Context, txs types.Transactions, blockNumber rpc.BlockNumber, minTimestamp uint64, maxTimestamp uint64) error + SendBundle(ctx context.Context, txs types.Transactions, blockNumber rpc.BlockNumber, minTimestamp uint64, maxTimestamp uint64, revertingTxHashes []common.Hash) error GetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error) GetPoolTransactions() (types.Transactions, error) GetPoolTransaction(txHash common.Hash) *types.Transaction diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index 4ef71b67e822..e4fef6a7d5aa 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -561,7 +561,7 @@ web3._extend({ new web3._extend.Method({ name: 'sendBundle', call: 'eth_sendBundle', - params: 4 + params: 1 }), ], properties: [ diff --git a/les/api_backend.go b/les/api_backend.go index 9491d3bfed88..2d14099fb4e6 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -197,8 +197,8 @@ func (b *LesApiBackend) SendTx(ctx context.Context, signedTx *types.Transaction) func (b *LesApiBackend) RemoveTx(txHash common.Hash) { b.eth.txPool.RemoveTx(txHash) } -func (b *LesApiBackend) SendBundle(ctx context.Context, txs types.Transactions, blockNumber rpc.BlockNumber, minTimestamp uint64, maxTimestamp uint64) error { - return b.eth.txPool.AddMevBundle(txs, big.NewInt(blockNumber.Int64()), minTimestamp, maxTimestamp) +func (b *LesApiBackend) SendBundle(ctx context.Context, txs types.Transactions, blockNumber rpc.BlockNumber, minTimestamp uint64, maxTimestamp uint64, revertingTxHashes []common.Hash) error { + return b.eth.txPool.AddMevBundle(txs, big.NewInt(blockNumber.Int64()), minTimestamp, maxTimestamp, revertingTxHashes) } func (b *LesApiBackend) GetPoolTransactions() (types.Transactions, error) { diff --git a/light/txpool.go b/light/txpool.go index c16be9b91660..f8563f91d3d6 100644 --- a/light/txpool.go +++ b/light/txpool.go @@ -558,6 +558,6 @@ func (pool *TxPool) MevBundles(blockNumber *big.Int, blockTimestamp uint64) ([]t } // AddMevBundle adds a mev bundle to the pool -func (pool *TxPool) AddMevBundle(txs types.Transactions, blockNumber *big.Int, minTimestamp uint64, maxTimestamp uint64) error { +func (pool *TxPool) AddMevBundle(txs types.Transactions, blockNumber *big.Int, minTimestamp uint64, maxTimestamp uint64, revertingTxHashes []common.Hash) error { return nil } diff --git a/miner/worker.go b/miner/worker.go index 8f8f1731dc57..5d28208db6ed 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -1190,11 +1190,11 @@ func (w *worker) commitNewWork(interrupt *int32, noempty bool, timestamp int64) log.Error("Failed to generate flashbots bundle", "err", err) return } - log.Info("Flashbots bundle", "ethToCoinbase", ethIntToFloat(bundle.totalEth), "gasUsed", bundle.totalGasUsed, "bundleScore", bundle.mevGasPrice, "bundleLength", len(bundle.txs)) - if len(bundle.txs) == 0 { + log.Info("Flashbots bundle", "ethToCoinbase", ethIntToFloat(bundle.totalEth), "gasUsed", bundle.totalGasUsed, "bundleScore", bundle.mevGasPrice, "bundleLength", len(bundle.originalBundle.Txs)) + if len(bundle.originalBundle.Txs) == 0 { return } - if w.commitBundle(bundle.txs, w.coinbase, interrupt) { + if w.commitBundle(bundle.originalBundle.Txs, w.coinbase, interrupt) { return } } @@ -1247,13 +1247,13 @@ func (w *worker) commit(uncles []*types.Header, interval func(), update bool, st } type simulatedBundle struct { - txs types.Transactions - mevGasPrice *big.Int - totalEth *big.Int - totalGasUsed uint64 + mevGasPrice *big.Int + totalEth *big.Int + totalGasUsed uint64 + originalBundle types.MevBundle } -func (w *worker) findMostProfitableBundle(bundles []types.Transactions, coinbase common.Address, parent *types.Block, header *types.Header, pendingTxs map[common.Address]types.Transactions) (simulatedBundle, error) { +func (w *worker) findMostProfitableBundle(bundles []types.MevBundle, coinbase common.Address, parent *types.Block, header *types.Header, pendingTxs map[common.Address]types.Transactions) (simulatedBundle, error) { maxBundle := simulatedBundle{mevGasPrice: new(big.Int)} for _, bundle := range bundles { @@ -1262,7 +1262,7 @@ func (w *worker) findMostProfitableBundle(bundles []types.Transactions, coinbase return simulatedBundle{}, err } gasPool := new(core.GasPool).AddGas(header.GasLimit) - if len(bundle) == 0 { + if len(bundle.Txs) == 0 { continue } simmed, err := w.computeBundleGas(bundle, parent, header, state, gasPool, pendingTxs) @@ -1280,22 +1280,30 @@ func (w *worker) findMostProfitableBundle(bundles []types.Transactions, coinbase return maxBundle, nil } +func containsHash(arr []common.Hash, match common.Hash) bool { + for _, elem := range arr { + if elem == match { + return true + } + } + return false +} + // Compute the adjusted gas price for a whole bundle // Done by calculating all gas spent, adding transfers to the coinbase, and then dividing by gas used -func (w *worker) computeBundleGas(bundle types.Transactions, parent *types.Block, header *types.Header, state *state.StateDB, gasPool *core.GasPool, pendingTxs map[common.Address]types.Transactions) (simulatedBundle, error) { +func (w *worker) computeBundleGas(bundle types.MevBundle, parent *types.Block, header *types.Header, state *state.StateDB, gasPool *core.GasPool, pendingTxs map[common.Address]types.Transactions) (simulatedBundle, error) { var totalGasUsed uint64 = 0 var tempGasUsed uint64 totalEth := new(big.Int) - for _, tx := range bundle { + for _, tx := range bundle.Txs { coinbaseBalanceBefore := state.GetBalance(w.coinbase) - receipt, err := core.ApplyTransaction(w.chainConfig, w.chain, &w.coinbase, gasPool, state, header, tx, &tempGasUsed, *w.chain.GetVMConfig()) if err != nil { return simulatedBundle{}, err } - if receipt.Status == types.ReceiptStatusFailed { - return simulatedBundle{}, errors.New("revert") + if receipt.Status == types.ReceiptStatusFailed && !containsHash(bundle.RevertingTxHashes, receipt.TxHash) { + return simulatedBundle{}, errors.New("failed tx") } totalGasUsed += receipt.GasUsed @@ -1329,10 +1337,10 @@ func (w *worker) computeBundleGas(bundle types.Transactions, parent *types.Block } return simulatedBundle{ - txs: bundle, - mevGasPrice: new(big.Int).Div(totalEth, new(big.Int).SetUint64(totalGasUsed)), - totalEth: totalEth, - totalGasUsed: totalGasUsed, + mevGasPrice: new(big.Int).Div(totalEth, new(big.Int).SetUint64(totalGasUsed)), + totalEth: totalEth, + totalGasUsed: totalGasUsed, + originalBundle: bundle, }, nil } From 84ccf8565747c749d77ed898d1af8c73da8f920a Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Fri, 9 Apr 2021 12:20:09 -0700 Subject: [PATCH 10/17] Add bundle merging with multiple workers --- .github/workflows/go.yml | 1 + cmd/geth/main.go | 1 + cmd/geth/usage.go | 1 + cmd/utils/flags.go | 7 ++ internal/ethapi/api.go | 2 +- miner/miner.go | 19 ++--- miner/multi_worker.go | 84 +++++++++++++------- miner/worker.go | 166 ++++++++++++++++++++++++++++----------- 8 files changed, 195 insertions(+), 86 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 3fc1f2ff8c68..82dd308b0ef0 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -53,6 +53,7 @@ jobs: uses: actions/checkout@v2 with: repository: flashbots/mev-geth-demo + ref: jason/v0.2 path: e2e - run: cd e2e && yarn install diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 1a27a3255adb..ec3974de58e9 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -125,6 +125,7 @@ var ( utils.MinerExtraDataFlag, utils.MinerRecommitIntervalFlag, utils.MinerNoVerfiyFlag, + utils.MinerMaxMergedBundles, utils.NATFlag, utils.NoDiscoverFlag, utils.DiscoveryV5Flag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index dea0b7c08a09..ead305b8fee7 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -188,6 +188,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.MinerExtraDataFlag, utils.MinerRecommitIntervalFlag, utils.MinerNoVerfiyFlag, + utils.MinerMaxMergedBundles, }, }, { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 7ed5907dba85..040f9d68b99e 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -473,6 +473,11 @@ var ( Usage: "Time interval to recreate the block being mined", Value: ethconfig.Defaults.Miner.Recommit, } + MinerMaxMergedBundles = cli.IntFlag{ + Name: "miner.maxmergedbundles", + Usage: "flashbots - The maximum amount of bundles to merge. The miner will run this many workers in parallel to calculate if the full block is more profitable with these additional bundles.", + Value: 3, + } MinerNoVerfiyFlag = cli.BoolFlag{ Name: "miner.noverify", Usage: "Disable remote sealing verification", @@ -1401,6 +1406,8 @@ func setMiner(ctx *cli.Context, cfg *miner.Config) { if ctx.GlobalIsSet(MinerNoVerfiyFlag.Name) { cfg.Noverify = ctx.GlobalBool(MinerNoVerfiyFlag.Name) } + + cfg.MaxMergedBundles = ctx.GlobalInt(MinerMaxMergedBundles.Name) } func setWhitelist(ctx *cli.Context, cfg *ethconfig.Config) { diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 68e69327e7c4..6178289ee730 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -2093,7 +2093,7 @@ func (s *PrivateTxBundleAPI) SendBundle(ctx context.Context, args SendBundleArgs for _, encodedTx := range args.Txs { tx := new(types.Transaction) - if err := rlp.DecodeBytes(encodedTx, tx); err != nil { + if err := tx.UnmarshalBinary(encodedTx); err != nil { return err } txs = append(txs, tx) diff --git a/miner/miner.go b/miner/miner.go index fc747c9ec1f3..0f81a5f26949 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -42,15 +42,16 @@ type Backend interface { // Config is the configuration parameters of mining. type Config struct { - Etherbase common.Address `toml:",omitempty"` // Public address for block mining rewards (default = first account) - Notify []string `toml:",omitempty"` // HTTP URL list to be notified of new work packages (only useful in ethash). - NotifyFull bool `toml:",omitempty"` // Notify with pending block headers instead of work packages - ExtraData hexutil.Bytes `toml:",omitempty"` // Block extra data set by the miner - GasFloor uint64 // Target gas floor for mined blocks. - GasCeil uint64 // Target gas ceiling for mined blocks. - GasPrice *big.Int // Minimum gas price for mining a transaction - Recommit time.Duration // The time interval for miner to re-create mining work. - Noverify bool // Disable remote mining solution verification(only useful in ethash). + Etherbase common.Address `toml:",omitempty"` // Public address for block mining rewards (default = first account) + Notify []string `toml:",omitempty"` // HTTP URL list to be notified of new work packages (only useful in ethash). + NotifyFull bool `toml:",omitempty"` // Notify with pending block headers instead of work packages + ExtraData hexutil.Bytes `toml:",omitempty"` // Block extra data set by the miner + GasFloor uint64 // Target gas floor for mined blocks. + GasCeil uint64 // Target gas ceiling for mined blocks. + GasPrice *big.Int // Minimum gas price for mining a transaction + Recommit time.Duration // The time interval for miner to re-create mining work. + Noverify bool // Disable remote mining solution verification(only useful in ethash). + MaxMergedBundles int } // Miner creates blocks and searches for proof-of-work values. diff --git a/miner/multi_worker.go b/miner/multi_worker.go index ea2b2e4f6299..f28fe167db54 100644 --- a/miner/multi_worker.go +++ b/miner/multi_worker.go @@ -7,74 +7,100 @@ import ( "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" ) type multiWorker struct { - regularWorker *worker - flashbotsWorker *worker + workers []*worker + regularWorker *worker } func (w *multiWorker) stop() { - w.regularWorker.stop() - w.flashbotsWorker.stop() + for _, worker := range w.workers { + worker.stop() + } } func (w *multiWorker) start() { - w.regularWorker.start() - w.flashbotsWorker.start() + for _, worker := range w.workers { + worker.start() + } } func (w *multiWorker) close() { - w.regularWorker.close() - w.flashbotsWorker.close() + for _, worker := range w.workers { + worker.close() + } } func (w *multiWorker) isRunning() bool { - return w.regularWorker.isRunning() || w.flashbotsWorker.isRunning() + for _, worker := range w.workers { + if worker.isRunning() { + return true + } + } + return false } func (w *multiWorker) setExtra(extra []byte) { - w.regularWorker.setExtra(extra) - w.flashbotsWorker.setExtra(extra) + for _, worker := range w.workers { + worker.setExtra(extra) + } } func (w *multiWorker) setRecommitInterval(interval time.Duration) { - w.regularWorker.setRecommitInterval(interval) - w.flashbotsWorker.setRecommitInterval(interval) + for _, worker := range w.workers { + worker.setRecommitInterval(interval) + } } func (w *multiWorker) setEtherbase(addr common.Address) { - w.regularWorker.setEtherbase(addr) - w.flashbotsWorker.setEtherbase(addr) + for _, worker := range w.workers { + worker.setEtherbase(addr) + } } func (w *multiWorker) enablePreseal() { - w.regularWorker.enablePreseal() - w.flashbotsWorker.enablePreseal() + for _, worker := range w.workers { + worker.enablePreseal() + } } func (w *multiWorker) disablePreseal() { - w.regularWorker.disablePreseal() - w.flashbotsWorker.disablePreseal() + for _, worker := range w.workers { + worker.disablePreseal() + } } func newMultiWorker(config *Config, chainConfig *params.ChainConfig, engine consensus.Engine, eth Backend, mux *event.TypeMux, isLocalBlock func(*types.Block) bool, init bool) *multiWorker { queue := make(chan *task) + regularWorker := newWorker(config, chainConfig, engine, eth, mux, isLocalBlock, init, &flashbotsData{ + isFlashbots: false, + queue: queue, + }) + + workers := []*worker{regularWorker} + + for i := 1; i <= config.MaxMergedBundles; i++ { + workers = append(workers, + newWorker(config, chainConfig, engine, eth, mux, isLocalBlock, init, &flashbotsData{ + isFlashbots: true, + queue: queue, + maxMergedBundles: i, + })) + } + + log.Info("creating multi worker", "config.MaxMergedBundles", config.MaxMergedBundles, "worker", len(workers)) return &multiWorker{ - regularWorker: newWorker(config, chainConfig, engine, eth, mux, isLocalBlock, init, &flashbotsData{ - isFlashbots: false, - queue: queue, - }), - flashbotsWorker: newWorker(config, chainConfig, engine, eth, mux, isLocalBlock, init, &flashbotsData{ - isFlashbots: true, - queue: queue, - }), + regularWorker: regularWorker, + workers: workers, } } type flashbotsData struct { - isFlashbots bool - queue chan *task + isFlashbots bool + queue chan *task + maxMergedBundles int } diff --git a/miner/worker.go b/miner/worker.go index 5d28208db6ed..b4c85cf04376 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -20,6 +20,7 @@ import ( "bytes" "errors" "math/big" + "sort" "sync" "sync/atomic" "time" @@ -39,7 +40,7 @@ import ( const ( // resultQueueSize is the size of channel listening to sealing result. - resultQueueSize = 10 + resultQueueSize = 20 // txChanSize is the size of channel listening to NewTxsEvent. // The number is referenced from the size of tx pool. @@ -103,6 +104,7 @@ type task struct { profit *big.Int isFlashbots bool + worker int } const ( @@ -620,7 +622,7 @@ func (w *worker) taskLoop() { // Interrupt previous sealing operation interrupt() stopCh, prev = make(chan struct{}), sealHash - log.Info("Proposed miner block", "blockNumber", task.block.Number(), "profit", ethIntToFloat(prevProfit), "isFlashbots", task.isFlashbots, "sealhash", sealHash, "parentHash", prevParentHash) + log.Info("Proposed miner block", "blockNumber", task.block.Number(), "profit", ethIntToFloat(prevProfit), "isFlashbots", task.isFlashbots, "sealhash", sealHash, "parentHash", prevParentHash, "worker", task.worker) if w.skipSealHook != nil && w.skipSealHook(task) { continue } @@ -803,9 +805,8 @@ func (w *worker) updateSnapshot() { w.snapshotState = w.current.state.Copy() } -func (w *worker) commitTransaction(tx *types.Transaction, coinbase common.Address, trackProfit bool) ([]*types.Log, error) { +func (w *worker) commitTransaction(tx *types.Transaction, coinbase common.Address) ([]*types.Log, error) { snap := w.current.state.Snapshot() - initialBalance := w.current.state.GetBalance(w.coinbase) receipt, err := core.ApplyTransaction(w.chainConfig, w.chain, &coinbase, w.current.gasPool, w.current.state, w.current.header, tx, &w.current.header.GasUsed, *w.chain.GetVMConfig()) if err != nil { @@ -815,14 +816,8 @@ func (w *worker) commitTransaction(tx *types.Transaction, coinbase common.Addres w.current.txs = append(w.current.txs, tx) w.current.receipts = append(w.current.receipts, receipt) - // coinbase balance difference already contains gas fee - if trackProfit { - finalBalance := w.current.state.GetBalance(w.coinbase) - w.current.profit.Add(w.current.profit, new(big.Int).Sub(finalBalance, initialBalance)) - } else { - gasUsed := new(big.Int).SetUint64(receipt.GasUsed) - w.current.profit.Add(w.current.profit, gasUsed.Mul(gasUsed, tx.GasPrice())) - } + gasUsed := new(big.Int).SetUint64(receipt.GasUsed) + w.current.profit.Add(w.current.profit, gasUsed.Mul(gasUsed, tx.GasPrice())) return receipt.Logs, nil } @@ -881,7 +876,7 @@ func (w *worker) commitBundle(txs types.Transactions, coinbase common.Address, i // Start executing the transaction w.current.state.Prepare(tx.Hash(), common.Hash{}, w.current.tcount) - logs, err := w.commitTransaction(tx, coinbase, true) + logs, err := w.commitTransaction(tx, coinbase) switch { case errors.Is(err, core.ErrGasLimitReached): // Pop the current out-of-gas transaction without shifting in the next from the account @@ -1000,7 +995,7 @@ func (w *worker) commitTransactions(txs *types.TransactionsByPriceAndNonce, coin // Start executing the transaction w.current.state.Prepare(tx.Hash(), w.current.tcount) - logs, err := w.commitTransaction(tx, coinbase, false) + logs, err := w.commitTransaction(tx, coinbase) switch { case errors.Is(err, core.ErrGasLimitReached): // Pop the current out-of-gas transaction without shifting in the next from the account @@ -1185,18 +1180,20 @@ func (w *worker) commitNewWork(interrupt *int32, noempty bool, timestamp int64) log.Error("Failed to fetch pending transactions", "err", err) return } - bundle, err := w.findMostProfitableBundle(bundles, w.coinbase, parent, header, pending) + + bundleTxs, bundle, numBundles, err := w.generateFlashbotsBundle(bundles, w.coinbase, parent, header, pending) if err != nil { log.Error("Failed to generate flashbots bundle", "err", err) return } - log.Info("Flashbots bundle", "ethToCoinbase", ethIntToFloat(bundle.totalEth), "gasUsed", bundle.totalGasUsed, "bundleScore", bundle.mevGasPrice, "bundleLength", len(bundle.originalBundle.Txs)) - if len(bundle.originalBundle.Txs) == 0 { + log.Info("Flashbots bundle", "ethToCoinbase", ethIntToFloat(bundle.totalEth), "gasUsed", bundle.totalGasUsed, "bundleScore", bundle.mevGasPrice, "bundleLength", len(bundleTxs), "numBundles", numBundles, "worker", w.flashbots.maxMergedBundles) + if len(bundleTxs) == 0 { return } - if w.commitBundle(bundle.originalBundle.Txs, w.coinbase, interrupt) { + if w.commitBundle(bundleTxs, w.coinbase, interrupt) { return } + w.current.profit.Add(w.current.profit, bundle.totalEth) } if len(localTxs) > 0 { txs := types.NewTransactionsByPriceAndNonce(w.current.signer, localTxs, header.BaseFee) @@ -1228,13 +1225,13 @@ func (w *worker) commit(uncles []*types.Header, interval func(), update bool, st interval() } select { - case w.taskCh <- &task{receipts: receipts, state: s, block: block, createdAt: time.Now(), profit: w.current.profit, isFlashbots: w.flashbots.isFlashbots}: + case w.taskCh <- &task{receipts: receipts, state: s, block: block, createdAt: time.Now(), profit: w.current.profit, isFlashbots: w.flashbots.isFlashbots, worker: w.flashbots.maxMergedBundles}: w.unconfirmed.Shift(block.NumberU64() - 1) log.Info("Commit new mining work", "number", block.Number(), "sealhash", w.engine.SealHash(block.Header()), "uncles", len(uncles), "txs", w.current.tcount, "gas", block.GasUsed(), "fees", totalFees(block, receipts), "profit", ethIntToFloat(w.current.profit), "elapsed", common.PrettyDuration(time.Since(start)), - "isFlashbots", w.flashbots.isFlashbots) + "isFlashbots", w.flashbots.isFlashbots, "worker", w.flashbots.maxMergedBundles) case <-w.exitCh: log.Info("Worker has exited") @@ -1247,37 +1244,102 @@ func (w *worker) commit(uncles []*types.Header, interval func(), update bool, st } type simulatedBundle struct { - mevGasPrice *big.Int - totalEth *big.Int - totalGasUsed uint64 - originalBundle types.MevBundle + mevGasPrice *big.Int + totalEth *big.Int + ethSentToCoinbase *big.Int + totalGasUsed uint64 + originalBundle types.MevBundle } -func (w *worker) findMostProfitableBundle(bundles []types.MevBundle, coinbase common.Address, parent *types.Block, header *types.Header, pendingTxs map[common.Address]types.Transactions) (simulatedBundle, error) { - maxBundle := simulatedBundle{mevGasPrice: new(big.Int)} +func (w *worker) generateFlashbotsBundle(bundles []types.MevBundle, coinbase common.Address, parent *types.Block, header *types.Header, pendingTxs map[common.Address]types.Transactions) (types.Transactions, simulatedBundle, int, error) { + simulatedBundles, err := w.simulateBundles(bundles, coinbase, parent, header, pendingTxs) + if err != nil { + return nil, simulatedBundle{}, 0, err + } + + sort.SliceStable(simulatedBundles, func(i, j int) bool { + return simulatedBundles[j].mevGasPrice.Cmp(simulatedBundles[i].mevGasPrice) < 0 + }) + + return w.mergeBundles(simulatedBundles, parent, header, pendingTxs) +} + +func (w *worker) mergeBundles(bundles []simulatedBundle, parent *types.Block, header *types.Header, pendingTxs map[common.Address]types.Transactions) (types.Transactions, simulatedBundle, int, error) { + finalBundle := types.Transactions{} + + state, err := w.chain.StateAt(parent.Root()) + if err != nil { + return nil, simulatedBundle{}, 0, err + } + gasPool := new(core.GasPool).AddGas(header.GasLimit) + + prevState := state + prevGasPool := gasPool + + mergedBundle := simulatedBundle{ + totalEth: new(big.Int), + ethSentToCoinbase: new(big.Int), + } + + count := 0 + for _, bundle := range bundles { + prevState = state.Copy() + prevGasPool = new(core.GasPool).AddGas(gasPool.Gas()) + + simmed, err := w.computeBundleGas(bundle.originalBundle, parent, header, state, gasPool, pendingTxs, len(finalBundle)) + if err != nil || simmed.totalEth.Cmp(new(big.Int)) <= 0 { + state = prevState + gasPool = prevGasPool + continue + } + + log.Info("Included bundle", "ethToCoinbase", ethIntToFloat(simmed.totalEth), "gasUsed", simmed.totalGasUsed, "bundleScore", simmed.mevGasPrice, "bundleLength", len(simmed.originalBundle.Txs), "worker", w.flashbots.maxMergedBundles) + + finalBundle = append(finalBundle, bundle.originalBundle.Txs...) + mergedBundle.totalEth.Add(mergedBundle.totalEth, simmed.totalEth) + mergedBundle.ethSentToCoinbase.Add(mergedBundle.ethSentToCoinbase, simmed.ethSentToCoinbase) + mergedBundle.totalGasUsed += simmed.totalGasUsed + count++ + + if count >= w.flashbots.maxMergedBundles { + break + } + } + + if len(finalBundle) == 0 || count != w.flashbots.maxMergedBundles { + return nil, simulatedBundle{}, count, nil + } + + return finalBundle, simulatedBundle{ + mevGasPrice: new(big.Int).Div(mergedBundle.totalEth, new(big.Int).SetUint64(mergedBundle.totalGasUsed)), + totalEth: mergedBundle.totalEth, + ethSentToCoinbase: mergedBundle.ethSentToCoinbase, + totalGasUsed: mergedBundle.totalGasUsed, + }, count, nil +} + +func (w *worker) simulateBundles(bundles []types.MevBundle, coinbase common.Address, parent *types.Block, header *types.Header, pendingTxs map[common.Address]types.Transactions) ([]simulatedBundle, error) { + simulatedBundles := []simulatedBundle{} for _, bundle := range bundles { state, err := w.chain.StateAt(parent.Root()) if err != nil { - return simulatedBundle{}, err + return nil, err } gasPool := new(core.GasPool).AddGas(header.GasLimit) if len(bundle.Txs) == 0 { continue } - simmed, err := w.computeBundleGas(bundle, parent, header, state, gasPool, pendingTxs) + simmed, err := w.computeBundleGas(bundle, parent, header, state, gasPool, pendingTxs, 0) if err != nil { log.Debug("Error computing gas for a bundle", "error", err) continue } - - if simmed.mevGasPrice.Cmp(maxBundle.mevGasPrice) > 0 { - maxBundle = simmed - } + simulatedBundles = append(simulatedBundles, simmed) } - return maxBundle, nil + return simulatedBundles, nil } func containsHash(arr []common.Hash, match common.Hash) bool { @@ -1291,13 +1353,17 @@ func containsHash(arr []common.Hash, match common.Hash) bool { // Compute the adjusted gas price for a whole bundle // Done by calculating all gas spent, adding transfers to the coinbase, and then dividing by gas used -func (w *worker) computeBundleGas(bundle types.MevBundle, parent *types.Block, header *types.Header, state *state.StateDB, gasPool *core.GasPool, pendingTxs map[common.Address]types.Transactions) (simulatedBundle, error) { +func (w *worker) computeBundleGas(bundle types.MevBundle, parent *types.Block, header *types.Header, state *state.StateDB, gasPool *core.GasPool, pendingTxs map[common.Address]types.Transactions, currentTxCount int) (simulatedBundle, error) { var totalGasUsed uint64 = 0 var tempGasUsed uint64 - totalEth := new(big.Int) + gasFees := new(big.Int) - for _, tx := range bundle.Txs { + ethSentToCoinbase := new(big.Int) + + for i, tx := range bundle.Txs { + state.Prepare(tx.Hash(), common.Hash{}, i+currentTxCount) coinbaseBalanceBefore := state.GetBalance(w.coinbase) + receipt, err := core.ApplyTransaction(w.chainConfig, w.chain, &w.coinbase, gasPool, state, header, tx, &tempGasUsed, *w.chain.GetVMConfig()) if err != nil { return simulatedBundle{}, err @@ -1314,8 +1380,8 @@ func (w *worker) computeBundleGas(bundle types.MevBundle, parent *types.Block, h } txInPendingPool := false - // check if tx is in pending pool if accountTxs, ok := pendingTxs[from]; ok { + // check if tx is in pending pool txNonce := tx.Nonce() for _, accountTx := range accountTxs { @@ -1326,21 +1392,27 @@ func (w *worker) computeBundleGas(bundle types.MevBundle, parent *types.Block, h } } - coinbaseBalanceAfter := state.GetBalance(w.coinbase) - totalEth = totalEth.Add(totalEth, coinbaseBalanceAfter.Sub(coinbaseBalanceAfter, coinbaseBalanceBefore)) - - if txInPendingPool { - // If tx is in pending pool, ignore the gas fees + if !txInPendingPool { + // If tx is not in pending pool, count the gas fees gasUsed := new(big.Int).SetUint64(receipt.GasUsed) - totalEth.Sub(totalEth, gasUsed.Mul(gasUsed, tx.GasPrice())) + gasFeesTx := gasUsed.Mul(gasUsed, tx.GasPrice()) + gasFees.Add(gasFees, gasFeesTx) + + coinbaseBalanceAfter := state.GetBalance(w.coinbase) + coinbaseDelta := big.NewInt(0).Sub(coinbaseBalanceAfter, coinbaseBalanceBefore) + coinbaseDelta.Sub(coinbaseDelta, gasFeesTx) + ethSentToCoinbase.Add(ethSentToCoinbase, coinbaseDelta) } } + totalEth := new(big.Int).Add(ethSentToCoinbase, gasFees) + return simulatedBundle{ - mevGasPrice: new(big.Int).Div(totalEth, new(big.Int).SetUint64(totalGasUsed)), - totalEth: totalEth, - totalGasUsed: totalGasUsed, - originalBundle: bundle, + mevGasPrice: new(big.Int).Div(totalEth, new(big.Int).SetUint64(totalGasUsed)), + totalEth: totalEth, + ethSentToCoinbase: ethSentToCoinbase, + totalGasUsed: totalGasUsed, + originalBundle: bundle, }, nil } From d141f058a9cc50ab9725a251478bba3c27f3af69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Kajetan=20Sta=C5=84czak?= Date: Mon, 12 Apr 2021 00:24:24 +0100 Subject: [PATCH 11/17] Update README.md --- .github/workflows/go.yml | 1 - MEV_spec_RPC_v0_1.md | 130 +++++++++++++++++++++++++++ MEV_spec_RPC_v0_2.md | 133 +++++++++++++++++++++++++++ MEV_spec_v0_1.md | 120 +++++++++++++++++++++++++ MEV_spec_v0_2.md | 184 ++++++++++++++++++++++++++++++++++++++ README.md | 188 ++------------------------------------- 6 files changed, 575 insertions(+), 181 deletions(-) create mode 100644 MEV_spec_RPC_v0_1.md create mode 100644 MEV_spec_RPC_v0_2.md create mode 100644 MEV_spec_v0_1.md create mode 100644 MEV_spec_v0_2.md diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 82dd308b0ef0..3fc1f2ff8c68 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -53,7 +53,6 @@ jobs: uses: actions/checkout@v2 with: repository: flashbots/mev-geth-demo - ref: jason/v0.2 path: e2e - run: cd e2e && yarn install diff --git a/MEV_spec_RPC_v0_1.md b/MEV_spec_RPC_v0_1.md new file mode 100644 index 000000000000..089b25adbe3d --- /dev/null +++ b/MEV_spec_RPC_v0_1.md @@ -0,0 +1,130 @@ +--- +tags: spec +--- + +# MEV-Geth RPC v0.1 + +# eth_sendBundle + +### Description + +Sends a bundle of transactions to the miner. The bundle has to be executed at the beginning of the block (before any other transactions), with bundle transactions executed exactly in the same order as provided in the bundle. During the Flashbots Alpha this is only called by the Flashbots relay. + +| Name | Type | Description | Comment +--------|----------|-----------|----------- +transactions | `Array` | Array of signed transactions (`eth_sendRawTransaction` style, signed and RLP-encoded) | a no-op in the light mode +blockNumber |`Quantity` |Exact block number at which the bundle can be included. |bundle is evicted after the block +minTimestamp |`Quantity` |Minimum (inclusive) block timestamp at which the bundle can be included. If this value is 0 then any timestamp is acceptable. +maxTimestamp |`Quantity` |Maximum (inclusive) block timestamp at which the bundle can be included. If this value is 0 then any timestamp is acceptable. + +### Returns + +{`boolean`} - `true` if bundle has been accepted by the node, otherwise `false` + +### Example + +```bash +# Request +curl -X POST --data '{ + "id": 1337, + "jsonrpc": "2.0", + "method": "eth_sendBundle", + "params": [ + [ + "f9014946843b9aca00830493e094a011e5f4ea471ee4341a135bb1a4af368155d7a280b8e40d5f2659000000000000000000000000fdd45a22dd1d606b3782f2119621e928e32743000000000000000000000000000000000000000000000000000000000077359400000000000000000000000000000000000000000000000", + "f86e8204d085012a05f200830c350094daf24c20717f428f00d8448d74d67a77f67ceb8287354a6ba7a18000802ea00e411bcb660dd8d47717df89078d2e8160c08e7f11cb7ad0ee935e7436eceb32a013ee00a21b7fa0a9f9c1224d11261648191875d4633aed6003543ea319f12b62" + ], + "0x12ab34", + "0x0", + "0x0" + ] +}' + +# Response +{ + "id": 1337, + "jsonrpc": "2.0", + "result": "true" +} +``` + +# eth_callBundle + +### Description + +Simulate a bundle of transactions at the top of a block. + +After retrieving the block specified in the `blockNrOrHash` it takes the same `blockhash`, `gasLimit`, `difficulty`, same `timestamp` unless the `blockTimestamp` property is specified, and increases the block number by `1`. `eth_callBundle` will timeout after `5` seconds. + +| Name | Type | Description | +| ---- | ---- | ----------- | +| encodedTxs | `Array` | Array of signed transactions (`eth_sendRawTransaction` style, signed and RLP-encoded) | +| blockNrOrHash | `Quantity\|string\|Block Identifier` | Block number, or one of "latest", "earliest" or "pending", or a block identifier as described in {Block Identifier} | +| blockTimestamp |`Quantity` |Block timestamp to be used in replacement of the timestamp taken from the parent block. | + +### Returns + +Map<`Data`, "error|value" : `Data`> - a mapping from transaction hashes to execution results with error or output (value) for each of the transactions + +### Example + +```bash +# Request +curl -X POST --data '{ + "id": 1337, + "jsonrpc": "2.0", + "method": "eth_callBundle", + "params": [ + [ + "f9014946843b9aca00830493e094a011e5f4ea471ee4341a135bb1a4af368155d7a280b8e40d5f2659000000000000000000000000fdd45a22dd1d606b3782f2119621e928e32743000000000000000000000000000000000000000000000000000000000077359400000000000000000000000000000000000000000000000", + "f86e8204d085012a05f200830c350094daf24c20717f428f00d8448d74d67a77f67ceb8287354a6ba7a18000802ea00e411bcb660dd8d47717df89078d2e8160c08e7f11cb7ad0ee935e7436eceb32a013ee00a21b7fa0a9f9c1224d11261648191875d4633aed6003543ea319f12b62" + ], + "0x12ab34" + ] +}' + +# Response +{ + "id": 1337, + "jsonrpc": "2.0", + "result": + { + "0x22b3806fbef9532db4105475222983404783aacd4d865ea5dab76a84aa1a07eb" : { + "value" : "0x0012" + }, + "0x489e3b5493af31d55059f8e296351b267720bc4ba7dc170871c1d789e5541027" : { + "value" : "0xabcd" + } + } +} +``` + +--- + +Below type description can also be found in [EIP-1474](https://eips.ethereum.org/EIPS/eip-1474) + +### `Quantity` + +- A `Quantity` value **MUST** be hex-encoded. +- A `Quantity` value **MUST** be "0x"-prefixed. +- A `Quantity` value **MUST** be expressed using the fewest possible hex digits per byte. +- A `Quantity` value **MUST** express zero as "0x0". + +### `Data` + +- A `Data` value **MUST** be hex-encoded. +- A `Data` value **MUST** be “0x”-prefixed. +- A `Data` value **MUST** be expressed using two hex digits per byte. + +### `Block Identifier` + +Since there is no way to clearly distinguish between a `Data` parameter and a `Quantity` parameter, [EIP-1898](https://eips.ethereum.org/EIPS/eip-1898) provides a format to specify a block either using the block hash or block number. The block identifier is a JSON `object` with the following fields: + +| Position | Name | Type | Description | +| -------- | ---- | ---- | ------------| +| 0A |blockNumber |`Quantity` |The block in the canonical chain with this number | +| 0B |blockHash |`Data` | The block uniquely identified by this hash. The blockNumber and blockHash properties are mutually exclusive; exactly one of them must be set. | +| 1B |requireCanonical |`boolean` | (optional) Whether or not to throw an error if the block is not in the canonical chain as described below. Only allowed in conjunction with the blockHash tag. Defaults to false. | + + +If the block is not found, the callee SHOULD raise a JSON-RPC error (the recommended error code is `-32001: Resource not found`. If the tag is `blockHash` and `requireCanonical` is `true`, the callee SHOULD additionally raise a JSON-RPC error if the block is not in the canonical chain (the recommended error code is `-32000: Invalid input` and in any case should be different than the error code for the block not found case so that the caller can distinguish the cases). The block-not-found check SHOULD take precedence over the block-is-canonical check, so that if the block is not found the callee raises block-not-found rather than block-not-canonical. \ No newline at end of file diff --git a/MEV_spec_RPC_v0_2.md b/MEV_spec_RPC_v0_2.md new file mode 100644 index 000000000000..2bae04de31c8 --- /dev/null +++ b/MEV_spec_RPC_v0_2.md @@ -0,0 +1,133 @@ +--- +tags: spec +--- + +# MEV-Geth RPC v0.2 + +# eth_sendBundle + +### Description + +Sends a bundle of transactions to the miner. The bundle has to be executed at the beginning of the block (before any other transactions), with bundle transactions executed exactly in the same order as provided in the bundle. During the Flashbots Alpha this is only called by the Flashbots relay. + +| Name | Type | Description | Comment +--------|----------|-----------|----------- +txs | `Array` | Array of signed transactions (`eth_sendRawTransaction` style, signed and RLP-encoded) | a no-op in the light mode +blockNumber |`Quantity` |Exact block number at which the bundle can be included. |bundle is evicted after the block +minTimestamp |`Quantity` |Minimum (inclusive) block timestamp at which the bundle can be included. If this value is 0 then any timestamp is acceptable. +maxTimestamp |`Quantity` |Maximum (inclusive) block timestamp at which the bundle can be included. If this value is 0 then any timestamp is acceptable. +revertingTxHashes |Array<`Data`> |Array of tx hashes within the bundle that are allowed to cause the EVM execution to revert without preventing the bundle inclusion in a block. + +### Returns + +{`boolean`} - `true` if bundle has been accepted by the node, otherwise `false` + +### Example + +```bash +# Request +curl -X POST --data '{ + "id": 1337, + "jsonrpc": "2.0", + "method": "eth_sendBundle", + "params": [ + { + "txs" : [ + "f9014946843b9aca00830493e094a011e5f4ea471ee4341a135bb1a4af368155d7a280b8e40d5f2659000000000000000000000000fdd45a22dd1d606b3782f2119621e928e32743000000000000000000000000000000000000000000000000000000000077359400000000000000000000000000000000000000000000000", + "f86e8204d085012a05f200830c350094daf24c20717f428f00d8448d74d67a77f67ceb8287354a6ba7a18000802ea00e411bcb660dd8d47717df89078d2e8160c08e7f11cb7ad0ee935e7436eceb32a013ee00a21b7fa0a9f9c1224d11261648191875d4633aed6003543ea319f12b62" + ], + "blockNumber" : "0x12ab34", + "minTimestamp" : "0x0", + "minTimestamp" :"0x0" + } + ] +}' + +# Response +{ + "id": 1337, + "jsonrpc": "2.0", + "result": "true" +} +``` + +# eth_callBundle + +### Description + +Simulate a bundle of transactions at the top of a block. + +After retrieving the block specified in the `blockNrOrHash` it takes the same `blockhash`, `gasLimit`, `difficulty`, same `timestamp` unless the `blockTimestamp` property is specified, and increases the block number by `1`. `eth_callBundle` will timeout after `5` seconds. + +| Name | Type | Description | +| ---- | ---- | ----------- | +| encodedTxs | `Array` | Array of signed transactions (`eth_sendRawTransaction` style, signed and RLP-encoded) | +| blockNrOrHash | `Quantity\|string\|Block Identifier` | Block number, or one of "latest", "earliest" or "pending", or a block identifier as described in {Block Identifier} | +| blockTimestamp |`Quantity` |Block timestamp to be used in replacement of the timestamp taken from the parent block. | + +### Returns + +Map<`Data`, "error|value" : `Data`> - a mapping from transaction hashes to execution results with error or output (value) for each of the transactions + +### Example + +```bash +# Request +curl -X POST --data '{ + "id": 1337, + "jsonrpc": "2.0", + "method": "eth_callBundle", + "params": [ + [ + "f9014946843b9aca00830493e094a011e5f4ea471ee4341a135bb1a4af368155d7a280b8e40d5f2659000000000000000000000000fdd45a22dd1d606b3782f2119621e928e32743000000000000000000000000000000000000000000000000000000000077359400000000000000000000000000000000000000000000000", + "f86e8204d085012a05f200830c350094daf24c20717f428f00d8448d74d67a77f67ceb8287354a6ba7a18000802ea00e411bcb660dd8d47717df89078d2e8160c08e7f11cb7ad0ee935e7436eceb32a013ee00a21b7fa0a9f9c1224d11261648191875d4633aed6003543ea319f12b62" + ], + "0x12ab34" + ] +}' + +# Response +{ + "id": 1337, + "jsonrpc": "2.0", + "result": + { + "0x22b3806fbef9532db4105475222983404783aacd4d865ea5dab76a84aa1a07eb" : { + "value" : "0x0012" + }, + "0x489e3b5493af31d55059f8e296351b267720bc4ba7dc170871c1d789e5541027" : { + "value" : "0xabcd" + } + } +} +``` + +--- + +Below type description can also be found in [EIP-1474](https://eips.ethereum.org/EIPS/eip-1474) + +### `Quantity` + +- A `Quantity` value **MUST** be hex-encoded. +- A `Quantity` value **MUST** be "0x"-prefixed. +- A `Quantity` value **MUST** be expressed using the fewest possible hex digits per byte. +- A `Quantity` value **MUST** express zero as "0x0". + +### `Data` + +- A `Data` value **MUST** be hex-encoded. +- A `Data` value **MUST** be “0x”-prefixed. +- A `Data` value **MUST** be expressed using two hex digits per byte. + +### `Block Identifier` + +Since there is no way to clearly distinguish between a `Data` parameter and a `Quantity` parameter, [EIP-1898](https://eips.ethereum.org/EIPS/eip-1898) provides a format to specify a block either using the block hash or block number. The block identifier is a JSON `object` with the following fields: + +| Position | Name | Type | Description | +| -------- | ---- | ---- | ------------| +| 0A |blockNumber |`Quantity` |The block in the canonical chain with this number | +| 0B |blockHash |`Data` | The block uniquely identified by this hash. The blockNumber and blockHash properties are mutually exclusive; exactly one of them must be set. | +| 1B |requireCanonical |`boolean` | (optional) Whether or not to throw an error if the block is not in the canonical chain as described below. Only allowed in conjunction with the blockHash tag. Defaults to false. | + + +If the block is not found, the callee SHOULD raise a JSON-RPC error (the recommended error code is `-32001: Resource not found`. If the tag is `blockHash` and `requireCanonical` is `true`, the callee SHOULD additionally raise a JSON-RPC error if the block is not in the canonical chain (the recommended error code is `-32000: Invalid input` and in any case should be different than the error code for the block not found case so that the caller can distinguish the cases). The block-not-found check SHOULD take precedence over the block-is-canonical check, so that if the block is not found the callee raises block-not-found rather than block-not-canonical. \ No newline at end of file diff --git a/MEV_spec_v0_1.md b/MEV_spec_v0_1.md new file mode 100644 index 000000000000..0e2a6ebb4f5e --- /dev/null +++ b/MEV_spec_v0_1.md @@ -0,0 +1,120 @@ +--- +tags: spec +--- + +# MEV-Geth v0.1 specification + +## Simple Summary + +Defines the construction and usage of MEV bundles by the miners. Provides specification for custom implementation of required node changes so that MEV bundles can be used correctly. + +## Abstract + +`MevBundles` are stored by the node and the best bundle is added to the block in front of other transactions. `MevBundles` are sorted by their `adjusted gas price`. + +## Motivation + +We believe that without the adoption of neutral, public, open-source infrastructure for permissionless MEV extraction, MEV risks becoming an insiders' game. We commit as an organisation to releasing reference implementations for participation in fair, ethical, and politically neutral MEV extraction. + +## Specification + +The key words `MUST`, `MUST NOT`, `REQUIRED`, `SHALL`, `SHALL NOT`, `SHOULD`, `SHOULD NOT`, `RECOMMENDED`, `MAY`, and `OPTIONAL` in this document are to be interpreted as described in [RFC-2119](https://www.ietf.org/rfc/rfc2119.txt). + +### Definitions + +#### `Bundle` +A set of transactions that `MUST` be executed together and `MUST` be executed at the beginning of the block. + +#### `Unit of work` +A `transaction`, a `bundle` or a `block`. + +#### `Subunit` +A discernible `unit of work` that is a part of a bigger `unit of work`. A `transaction` is a `subunit` of a `bundle` or a `block`. A `bundle` is a `subunit` of a `block`. + +#### `Total gas used` +A sum of gas units used by each transaction from the `unit of work`. + +#### `Average gas price` +Sum of (`gas price` * `total gas used`) of all `subunits` divided by the `total gas used` of the unit. + +#### `Direct coinbase payment` +A value of a transaction with a recipient set to be the same as the `coinbase` address. + +#### `Contract coinbase payment` +A payment from a smart contract to the `coinbase` address. + +#### `Profit` +A difference between the balance of the `coinbase` account at the end and at the beginning of the execution of a `unit of work`. We can measure a `transaction profit`, a `bundle profit`, and a `block profit`. + +Balance of the `coinbase` account changes in the following way +|Unit of work|Balance Change| +|-|-| +|Transaction| `average gas price` * `total gas used` + `direct coinbase payment` + `contract coinbase payment` | +|Bundle | `average gas price` * `total gas used` + `direct coinbase payment` + `contract coinbase payment` | +|Block | `block reward` + `average gas price` * `total gas used` + `direct coinbase payment` + `contract coinbase payment` | + +#### `Adjusted gas price` +`Unit of work` `profit` divided by the `total gas used` by the `unit of work`. + +#### `MevBundle` +An object with four properties: + +|Property| Type|Description| +|-|-|-| +|`transactions`|`Array`|A list of transactions in the bundle. Each transaction is signed and RLP-encoded.| +|`blockNumber`|`uint64`|The exact block number at which the bundle can be executed| +|`minTimestamp`|`uint64`|Minimum block timestamp (inclusive) at which the bundle can be executed| +|`maxTimestamp`|`uint64`|Maximum block timestamp (inclusive) at which the bundle can be executed| + +### Bundle construction + +Bundle `SHOULD` contain transactions with nonces that are following the current nonces of the signing addresses or other transactions preceding them in the same bundle. + +A bundle `MUST` contain at least one transaction. There is no upper limit for the number of transactions in the bundle, however bundles that exceed the block gas limit will always be rejected. + +A bundle `MAY` include a `direct coinbase payment` or a `contract coinbase payment`. Bundles that do not contain such payments may lose comparison when their `profit` is compared with other bundles. + +The `maxTimestamp` value `MUST` be greater or equal the `minTimestamp` value. + +### Accepting bundles from the network + +Node `MUST` provide a way of exposing a JSON RPC endpoint accepting `eth_sendBundle` calls (specified [here](MEV_spec_RPC_v0_1.md)). Such endpoint `SHOULD` only be accepting calls from `MEV-relay` but there is no requirement to restrict it through the node source code as it can be done on the infrastructure level. + +### Bundle eligibility + +Any bundle that is correctly constructed `MUST` have a `blockNumber` field set which specifies in which block it can be included. If the node has already progressed to a later block number then such bundle `MAY` be removed from memory. + +Any bundle that is correctly constructed `MAY` have a `minTimestamp` and/or a `maxTimestamp` field set. Default values for both of these fields are `0` and the meaning of `0` is that any block timestamp value is accepted. When these values are not `0`, then `block.timestamp` is compared with them. If the current `block.timestamp` is greater than the `maxTimestamp` then the bundle `MUST NOT` be included in the block and `MAY` be removed from memory. If the `block.timestamp` is less than `minTimestamp` then the bundle `MUST NOT` be included in the block and `SHOULD NOT` be removed from memory (it awaits future blocks). + +### Block construction + +A block `MUST` either contain one bundle or no bundles. When a bundle is included it `MUST` be the bundle with the highest `adjusted gas price` among eligible bundles. The node `SHOULD` be able to compare a `block profit` in cases when a bundle is included (MEV block) and when no bundles are included (regular block) and choose a block with the highest `profit`. + +A block with a bundle `MUST` place the bundle at the beginning of the block and `MUST NOT` insert any transactions between the bundle transactions. + +### Bundle eviction + +Node `SHOULD` be able to limit the number of bundles kept in memory and apply an algorithm for selecting bundles to be evicted when too many eligible bundles have been received. + +## Rationale + +### At most one MevBundle gets included in the block + +There are two reasons for which multiple bundles in a block may cause problems: + +- two bundles may affect each other's `profit` and so the bundle creator may not be willing to accept a possibility of not being added in the front of the block +- simulating multiple bundle combinations may be very straining for the node infrastructure and introduce excessive latency into the block creation process + +Both of these problems may be addressed in the future versions. + +## Each bundle needs a blockNumber + +This allows specifying bundles to be included in the future blocks (e.g. just after some smart contracts change their state). This cannot be used to ensure a specific parent block / hash. + +## Backwards Compatibility + +This change is not affecting consensus and is fully backwards compatible. + +## Security Considerations + +`MevBundles` that are awaiting future blocks must be stored by the miner's node and it is important to ensure that there is a mechanism to ensure that the storage is limits are not exceeded (whether they are store in memory or persisted). \ No newline at end of file diff --git a/MEV_spec_v0_2.md b/MEV_spec_v0_2.md new file mode 100644 index 000000000000..c912b8998520 --- /dev/null +++ b/MEV_spec_v0_2.md @@ -0,0 +1,184 @@ +--- +tags: spec +--- + +# MEV-Geth v0.2 specification + +## Simple Summary + +Defines the construction and usage of MEV bundles by miners. Provides a specification for custom implementations of the required node changes so that MEV bundles can be used correctly. + +## Abstract + +`MEV bundles` are stored by the node and the bundles that are providing extra profit for miners are added to the block in front of other transactions. + +## Motivation + +We believe that without the adoption of neutral, public, open-source infrastructure for permissionless MEV extraction, MEV risks becoming an insiders' game. We commit as an organisation to releasing reference implementations for participation in fair, ethical, and politically neutral MEV extraction. + +## Specification + +The key words `MUST`, `MUST NOT`, `REQUIRED`, `SHALL`, `SHALL NOT`, `SHOULD`, `SHOULD NOT`, `RECOMMENDED`, `MAY`, and `OPTIONAL` in this document are to be interpreted as described in [RFC-2119](https://www.ietf.org/rfc/rfc2119.txt). + +### Miner Configuration + +Miner should accept the following configuration options: +* miner.strictprofitswitch (int) - time in miliseconds to wait for both the non-MEV (vanilla) and the `MEV block` construction before selecting the best available block for mining. If value is zero then no waiting is necessary. +* miner.maxmergedbundles (int) - max number of `MEV bundles` to be included within a single block +* relayWSURL (string) - address of the relay WS endpoint +* relayWSSigningKey (bytes32) - a signing key for communication with the relay's WebSockets endpoint. The key should be a valid private key from a secp256k1 based ECDSA. +* relayWSSigningKeystoreFile (string) - path to the keystore file with the WS signing key (used when signing key not provided) +* relayWSKeystorePW (string) - password to unlock the relay WS keystore file + +### Definitions + +#### `Relay` + +An external system delivering `MEV bundles` to the node. Relay provides protection against DoS attacks. + +#### `MEV bundle` or `bundle` + +A list of transactions that `MUST` be executed together and in the same order as provided in the bundle, `MUST` be executed before any non-bundle transactions and only after other bundles that have a higher `bundle adjusted gas price`. + +Transactions in the bundle `MUST` execute without failure (return status 1 on transaction receipts) unless their hashes are included in the `revertingTxHashes` list. + +When representing a bundle in communication between the `relay` and the node we use an object with the following properties: + +|Property| Type|Description| +|-|-|-| +|`txs`|`Array`|A list of transactions in the bundle. Each transaction is signed and RLP-encoded.| +|`blockNumber`|`uint64`|The exact block number at which the bundle can be executed| +|`minTimestamp`|`uint64`|Minimum block timestamp (inclusive) at which the bundle can be executed| +|`maxTimestamp`|`uint64`|Maximum block timestamp (inclusive) at which the bundle can be executed| +|`revertingTxHashes`|`Array`|List of hashes of transactions that are allowed to return status 0 on transaction receipts| + +#### `MEV block` + +A block containing more than zero 'MEV bundles'. + +Whenever we say that a block contains a `bundle` we mean that the block includes all transactions of that bundle in the same order as in the `bundle`. + +#### `Unit of work` +A `transaction`, a `bundle` or a `block`. + +#### `Subunit` +A discernible `unit of work` that is a part of a bigger `unit of work`. A `transaction` is a `subunit` of a `bundle` or a `block`. A `bundle` is a `subunit` of a `block`. + +#### `Total gas used` +The sum of gas units used by each transaction from the `unit of work`. + +#### `Average gas price` +For a transaction it is equivalent to the transaction gas price and for other `units of work` it is a sum of (`average gas price` * `total gas used`) of all `subunits` divided by the `total gas used` by the unit. + +#### `Direct coinbase payment` +The value of a transaction with a recipient set to be the same as the `coinbase` address. + +#### `Contract coinbase payment` +A payment from a smart contract to the `coinbase` address. + +#### `Coinbase payment` +A sum of all `direct coinbase payments` and `contract coinbase payments` within the `unit of work`. + +#### `Eligible coinbase payment` +A sum of all `direct coinbase payments` and `contract coinbase payments` within the `unit of work`. + +#### `Gas fee payment` +An `average gas price` * `total gas used` within the `unit of work`. + +#### `Eligible gas fee payment` +A `gas fee payment` excluding `gas fee payments` from transactions that can be spotted by the miner in the publicly visible transaction pool. + +#### `Bundle scoring profit` +A sum of all `eligible coinbase payments` and `eligible gas payments` of a `bundle`. + +#### `Profit` +A difference between the balance of the `coinbase` account at the end and at the beginning of the execution of a `unit of work`. We can measure a `transaction profit`, a `bundle profit`, and a `block profit`. + +Balance of the `coinbase` account changes in the following way +|Unit of work|Balance Change| +|-|-| +|Transaction| `average gas price` * `total gas used` + `direct coinbase payment` + `contract coinbase payment` | +|Bundle | `average gas price` * `total gas used` + `direct coinbase payment` + `contract coinbase payment` | +|Block | `block reward` + `average gas price` * `total gas used` + `direct coinbase payment` + `contract coinbase payment` | + +#### `Adjusted gas price` +`Unit of work` `profit` divided by the `total gas used` by the `unit of work`. + +#### `Bundle adjusted gas price` +`Bundle scoring profit` divided by the `total gas used` by the `bundle`. + +### Bundle construction + +A bundle `SHOULD` contain transactions with nonces that are following the current nonces of the signing addresses or other transactions preceding them in the same bundle. + +A bundle `MUST` contain at least one transaction. There is no upper limit for the number of transactions in the bundle, however bundles that exceed the block gas limit will always be rejected. + +A bundle `MAY` include `eligible coinbase payments`. Bundles that do not contain such payments may be discarded when their `bundle adjusted gas price` is compared with other bundles. + +The `maxTimestamp` value `MUST` be greater or equal the `minTimestamp` value. + +### Accepting bundles from the network + +#### JSON RPC + +Node `MUST` provide a way of exposing a JSON RPC endpoint accepting `eth_sendBundle` calls (specified [here](MEV_spec_RPC_v0_2.md)). Such endpoint `SHOULD` only be accepting calls from the `relay` but there is no requirement to restrict it through the node source code as it can be done on the infrastructure level. + +#### WebSockets + +Node `MUST` be able to connect to the relay WebSocket endpoint provided as a configuration option named `RelayWSURL` using an authentication key and maintain an open connection. Authentication key `MUST` be provided with the key security in mind. + +During the WebSocket connection initiation and on each reconnection, the node `MUST` sign a timestamp (unix epoch in seconds, UTC) using the `RelayWSSigningKey` and send it in the request header body (X-Auth-Message). The X-Auth-Message should be a JSON encoded object with three properties called timestamp, signature and coinbase. + +### Bundle eligibility + +Any bundle that is correctly constructed `MUST` have a `blockNumber` field set which specifies in which block it can be included. If the node has already progressed to a later block number then such bundle `MAY` be removed from memory. + +Any bundle that is correctly constructed `MAY` have a `minTimestamp` and/or a `maxTimestamp` field set. Default values for both of these fields are `0` and the meaning of `0` is that any block timestamp value is accepted. When these values are not `0`, then `block.timestamp` is compared with them. If the current `block.timestamp` is greater than the `maxTimestamp` then the bundle `MUST NOT` be included in the block and `MAY` be removed from memory. If the `block.timestamp` is less than `minTimestamp` then the bundle `MUST NOT` be included in the block and `SHOULD NOT` be removed from memory (it awaits future blocks). + +### Block construction + +`MEV bundles` `MUST` be sorted by their `bundle adjusted gas price` first and then one by one added to the block as long as there is any gas left in the block and the number of bundles added is less or equal the `MaxMergedBundles` parameter. The remaining block gas `SHOULD` be used for non-MEV transactions. + +Block `MUST` contain between 0 and `MaxMergedBundles` bundles. + +A block with bundles `MUST` place the bundles at the beginning of the block and `MUST NOT` insert any transactions between the bundles or bundle transactions. + +When constructing the block the node should reject any bundle that has a reverting transaction unless its hash is included in the `RevertingTxHashes` list of the bundle object. + +The node `SHOULD` be able to compare the `block profit` for each number of bundles between 0 and `MaxMergedBundles` and choose a block with the highest `profit`, e.g. if `MaxMergedBundles` is 3 then the node `SHOULD` build 4 different blocks - with the maximum of respectively 0, 1, 2, and 3 bundles and choose the one with the highest `profit`. + +When constructing blocks, if the 0 bundles block has not yet been constructed and the `StrictProfitSwitch` parameter is set to a value other than 0, then the node `MUST` wait `StrictProfitSwitch` miliseconds before accepting any MEV block for mining. + +### Bundle eviction + +Node `SHOULD` be able to limit the number of bundles kept in memory and apply an algorithm for selecting bundles to be evicted when too many eligible bundles have been received. + +## Rationale + +### Naive bundle merging + +The bundle merging process is not necessarily picking the most profitable combination of bundles but only the best guess achievable without degrading latency. The first bundle included is always the bundle with the highest `bundle adjusted gas price` + +### Using bundle adjusted gas price instead of adjusted gas price + +The `bundle adjusted gas price` is used to prevent bundle creators from artificially increasing the `adjusted gas price` by adding unrelated high gas price transactions from the publicly visible transaction pool. + +### Each bundle needs a blockNumber + +This allows specifying bundles to be included in the future blocks (e.g. just after some smart contracts change their state). This cannot be used to ensure a specific parent block / hash. + +## Future Considerations + +### Full block submission + +A proposal to allow MEV-Geth accepting fully constructed blocks as well as bundles is considered for inclusion in next versions. + +## Backwards Compatibility + +This change is not affecting consensus and is fully compatible with Ethereum specification. + +Bundle formats are not backwards compatible and the v0.2 bundles would be rejected by v0.1 MEV clients. + +## Security Considerations + +`MEV bundles` that are awaiting future blocks must be stored by the node to ensure that the storage limits are not exceeded (whether they are store in memory or persisted). \ No newline at end of file diff --git a/README.md b/README.md index 324de818f5be..9cf6a871b9fd 100644 --- a/README.md +++ b/README.md @@ -6,195 +6,23 @@ Flashbots is a research and development organization formed to mitigate the nega ## Quick start -<<<<<<< HEAD - -## Building the source - -For prerequisites and detailed build instructions please read the [Installation Instructions](https://geth.ethereum.org/docs/install-and-build/installing-geth). - -Building `geth` requires both a Go (version 1.14 or later) and a C compiler. You can install -them using your favourite package manager. Once the dependencies are installed, run - -```shell - +``` git clone https://github.com/flashbots/mev-geth cd mev-geth - -> > > > > > > dfdcfc666 (Add infra/CI and update README) -> > > > > > > make geth - +make geth ``` See [here](https://geth.ethereum.org/docs/install-and-build/installing-geth#build-go-ethereum-from-source-code) for further info on building MEV-geth from source. -## MEV-Geth: a proof of concept - -We have designed and implemented a proof of concept for permissionless MEV extraction called MEV-Geth. It is a sealed-bid block space auction mechanism for communicating transaction order preference. While our proof of concept has incomplete trust guarantees, we believe it's a significant improvement over the status quo. The adoption of MEV-Geth should relieve a lot of the network and chain congestion caused by frontrunning and backrunning bots. - -| Guarantee | PGA | Dark-txPool | MEV-Geth | -| -------------------- | :-: | :---------: | :------: | -| Permissionless | ✅ | ❌ | ✅ | -| Efficient | ❌ | ❌ | ✅ | -| Pre-trade privacy | ❌ | ✅ | ✅ | -| Failed trade privacy | ❌ | ❌ | ✅ | -| Complete privacy | ❌ | ❌ | ❌ | -| Finality | ❌ | ❌ | ❌ | - -### Why MEV-Geth? - -We believe that without the adoption of neutral, public, open-source infrastructure for permissionless MEV extraction, MEV risks becoming an insiders' game. We commit as an organization to releasing reference implementations for participation in fair, ethical, and politically neutral MEV extraction. By doing so, we hope to prevent the properties of Ethereum from being eroded by trust-based dark pools or proprietary channels which are key points of security weakness. We thus release MEV-Geth with the dual goal of creating an ecosystem for MEV extraction that preserves Ethereum properties, as well as starting conversations with the community around our research and development roadmap. - -### Design goals - -- **Permissionless** - A permissionless design implies there are no trusted intermediary which can censor transactions. -- **Efficient** - An efficient design implies MEV extraction is performed without causing unnecessary network or chain congestion. -- **Pre-trade privacy** - Pre-trade privacy implies transactions only become publicly known after they have been included in a block. Note, this type of privacy does not exclude privileged actors such as transaction aggregators / gateways / miners. -- **Failed trade privacy** - Failed trade privacy implies loosing bids are never included in a block, thus never exposed to the public. Failed trade privacy is tightly coupled to extraction efficiency. -- **Complete privacy** - Complete privacy implies there are no privileged actors such as transaction aggregators / gateways / miners who can observe incoming transactions. -- **Finality** - Finality implies it is infeasible for MEV extraction to be reversed once included in a block. This would protect against time-bandit chain re-org attacks. - -The MEV-Geth proof of concept relies on the fact that searchers can withhold bids from certain miners in order to disincentivize bad behavior like stealing a profitable strategy. We expect a complete privacy design to necessitate some sort of private computation solution like SGX, ZKP, or MPC to withhold the transaction content from miners until it is mined in a block. One of the core objective of the Flashbots organization is to incentivize and produce research in this direction. - -The MEV-Geth proof of concept does not provide any finality guarantees. We expect the solution to this problem to require post-trade execution privacy through private chain state or strong economic infeasibility. The design of a system with strong finality is the second core objective of the MEV-Geth research effort. - -<<<<<<< HEAD -This command will: - -- Start `geth` in fast sync mode (default, can be changed with the `--syncmode` flag), - causing it to download more data in exchange for avoiding processing the entire history - of the Ethereum network, which is very CPU intensive. -- Start up `geth`'s built-in interactive [JavaScript console](https://geth.ethereum.org/docs/interface/javascript-console), - (via the trailing `console` subcommand) through which you can interact using [`web3` methods](https://web3js.readthedocs.io/en/) - (note: the `web3` version bundled within `geth` is very old, and not up to date with official docs), - as well as `geth`'s own [management APIs](https://geth.ethereum.org/docs/rpc/server). - This tool is optional and if you leave it out you can always attach to an already running - `geth` instance with `geth attach`. - -### A Full node on the Görli test network - -Transitioning towards developers, if you'd like to play around with creating Ethereum -contracts, you almost certainly would like to do that without any real money involved until -you get the hang of the entire system. In other words, instead of attaching to the main -network, you want to join the **test** network with your node, which is fully equivalent to -the main network, but with play-Ether only. - -```shell -$ geth --goerli console -``` - -======= - -### How it works - -> > > > > > > dfdcfc666 (Add infra/CI and update README) - -MEV-Geth introduces the concepts of "searchers", "transaction bundles", and "block template" to Ethereum. Effectively, MEV-Geth provides a way for miners to delegate the task of finding and ordering transactions to third parties called "searchers". These searchers compete with each other to find the most profitable ordering and bid for its inclusion in the next block using a standardized template called a "transaction bundle". These bundles are evaluated in a sealed-bid auction hosted by miners to produce a "block template" which holds the [information about transaction order required to begin mining](https://ethereum.stackexchange.com/questions/268/ethereum-block-architecture). - -![](https://hackmd.io/_uploads/B1fWz7rcD.png) - -The MEV-Geth proof of concept is compatible with any regular Ethereum client. The Flashbots core devs are maintaining [a reference implementation](https://github.com/flashbots/mev-geth) for the go-ethereum client. - -### Differences between MEV-Geth and [_vanilla_ geth](https://github.com/ethereum/go-ethereum) +## Documentation -The entire patch can be broken down into four modules: +See [here](https://docs.flashbots.net) for Flashbots documentation. -1. bundle worker and `eth_sendBundle` rpc (commits [8104d5d7b0a54bd98b3a08479a1fde685eb53c29](https://github.com/flashbots/mev-geth/commit/8104d5d7b0a54bd98b3a08479a1fde685eb53c29) and [c2b5b4029b2b748a6f1a9d5668f12096f096563d](https://github.com/flashbots/mev-geth/commit/c2b5b4029b2b748a6f1a9d5668f12096f096563d)) -2. profit switcher (commit [aa5840d22f4882f91ecba0eb20ef35a702b134d5](https://github.com/flashbots/mev-geth/commit/aa5840d22f4882f91ecba0eb20ef35a702b134d5)) -3. `eth_callBundle` simulation rpc (commits [9199d2e13d484df7a634fad12343ed2b46d5d4c3](https://github.com/flashbots/mev-geth/commit/9199d2e13d484df7a634fad12343ed2b46d5d4c3) and [a99dfc198817dd171128cc22439c81896e876619](https://github.com/flashbots/mev-geth/commit/a99dfc198817dd171128cc22439c81896e876619)) -4. Documentation (this file) and CI/infrastructure configuration (commit [035109807944f7a446467aa27ca8ec98d109a465](https://github.com/flashbots/mev-geth/commit/035109807944f7a446467aa27ca8ec98d109a465)) - -The entire changeset can be viewed inspecting the [diff](https://github.com/ethereum/go-ethereum/compare/master...flashbots:master). - -In summary: - -- Geth’s txpool is modified to also contain a `mevBundles` field, which stores a list of MEV bundles. Each MEV bundle is an array of transactions, along with a min/max timestamp for their inclusion. -- A new `eth_sendBundle` API is exposed which allows adding an MEV Bundle to the txpool. During the Flashbots Alpha, this is only called by MEV-relay. - - The transactions submitted to the bundle are “eth_sendRawTransaction-style” RLP encoded signed transactions along with the min/max block of inclusion - - This API is a no-op when run in light mode -- Geth’s miner is modified as follows: - - While in the event loop, before adding all the pending txpool “normal” transactions to the block, it: - - Finds the most profitable bundle - - It picks the most profitable bundle by returning the one with the highest average gas price per unit of gas - - computeBundleGas: Returns average gas price (\sum{gasprice_i\*gasused_i + (coinbase_after - coinbase_before)) / \sum{gasused_i}) - - Commits the bundle (remember: Bundle transactions are not ordered by nonce or gas price). For each transaction in the bundle, it: - - `Prepare`’s it against the state - - CommitsTransaction with trackProfit = true - w.current.profit += coinbase_after_tx - coinbase_before_tx - w.current.profit += gas \* gas_price - - If a block is found where the w.current.profit is more than the previous profit, it switches mining to that block. -- A new `eth_callBundle` API is exposed that enables simulation of transaction bundles. -- Documentation and CI/infrastructure files are added. - -### MEV-Geth for miners - -Miners can start mining MEV blocks by running MEV-Geth, or by implementing their own fork that matches the specification. - -While only the bundle worker and `eth_sendBundle` module (1) is necessary to mine flashbots blocks, we recommend also running the profit switcher module (2) to guarantee mining rewards are maximized. The `eth_callBundle` simulation rpc module (3) is not needed for the alpha. The suggested configuration is implemented in the `master` branch of this repository, which also includes the documentation module (4). - -We issue and maintain [releases](https://github.com/flashbots/mev-geth/releases) for the recommended configuration for the current and immediately prior versions of geth. - -In order to see the diff of the recommended patch, run: - -``` - git diff master~4..master~1 -``` - -Alternatively, the `master-barebones` branch includes only modules (1) and (4), leaving the profit switching logic to miners. While this usage is discouraged, it entails a much smaller change in the code. - -At this stage, we recommend only receiving bundles via a relay, to prevent abuse via denial-of-service attacks. We have [implemented](https://github.com/flashbots/mev-relay) and currently run such relay. This relay performs basic rate limiting and miner profitability checks, but does otherwise not interfere with submitted bundles in any way, and is open for everybody to participate. We invite you to try the [Flashbots Alpha](https://github.com/flashbots/pm#flashbots-alpha) and start receiving MEV revenue by following these steps: - -1. Fill out this [form](https://forms.gle/78JS52d22dwrgabi6) to indicate your interest in participating in the Alpha and be added to the MEV-Relay miner whitelist. -2. You will receive an onboarding email from Flashbots to help [set up](https://github.com/flashbots/mev-geth/blob/master/README.md#quick-start) your MEV-Geth node and protect it with a [reverse proxy](https://github.com/flashbots/mev-relay-js/blob/master/miner/proxy.js) to open the `eth_sendBundle` RPC. -3. Respond to Flashbots' email with your MEV-Geth node endpoint to be added to the Flashbots hosted [MEV-relay](https://github.com/flashbots/mev-relay-js) gateway. MEV-Relay is needed during the alpha to aggregate bundle requests from all users, prevent spam and DOS attacks on participating miner(s)/mining pool(s), and collect system health metrics. -4. After receiving a confirmation email that your MEV-Geth node's endpoint has been added to the relay, you will immediately start receiving Flashbots transaction bundles with associated MEV revenue paid to you. - -### MEV-Geth for searchers - -You do _not_ need to run MEV-Geth as a searcher, but, instead, to monitor the Ethereum state and transaction pool for MEV opportunities and produce transaction bundles that extract that MEV. Anyone can become a searcher. In fact, the bundles produced by searchers don't need to extract MEV at all, but we expect the most valuable bundles will. - -An MEV-Geth bundle is a standard message template composed of an array of valid ethereum transactions, a blockheight, and an optional timestamp range over which the bundle is valid. - -```json -{ - "signedTransactions": ["..."], // RLP encoded signed transaction array - "blocknumber": "0x386526", // hex string - "minTimestamp": 12345, // optional uint64 - "maxTimestamp": 12345 // optional uint64 -} -``` - -The `signedTransactions` can be any valid ethereum transactions. Care must be taken to place transaction nonces in correct order. - -The `blocknumber` defines the block height at which the bundle is to be included. A bundle will only be evaluated for the provided blockheight and immediately evicted if not selected. - -The `minTimestamp` and `maxTimestamp` are optional conditions to further restrict bundle validity within a time range. - -<<<<<<< HEAD -If you'd like to contribute to go-ethereum, please fork, fix, commit and send a pull request -for the maintainers to review and merge into the main code base. If you wish to submit -more complex changes though, please check up with the core devs first on [our Discord Server](https://discord.gg/invite/nthXNEv) -to ensure those changes are in line with the general philosophy of the project and/or get -some early feedback which can make both your efforts much lighter as well as our review -and merge procedures quick and simple. -======= -MEV-Geth miners select the most profitable bundle per unit of gas used and place it at the beginning of the list of transactions of the block template at a given blockheight. Miners determine the value of a bundle based on the following equation. _Note, the change in block.coinbase balance represents a direct transfer of ETH through a smart contract._ - -> > > > > > > fcac1062f (Add infra/CI and update README) - - - -To submit a bundle, the searcher sends the bundle directly to the miner using the rpc method `eth_sendBundle`. Since MEV-Geth requires direct communication between searchers and miners, a searcher can configure the list of miners where they want to send their bundle. +| Version | Spec | +| ------- | ------------------------------------------------------------------------------------------- | +| v0.2 | [MEV-Geth Spec v0.2](https://docs.flashbots.net/flashbots-auction/miners/mev-geth-spec/v02) | +| v0.1 | [MEV-Geth Spec v0.1](https://docs.flashbots.net/flashbots-auction/miners/mev-geth-spec/v01) | ### Feature requests and bug reports If you are a user of MEV-Geth and have suggestions on how to make integration with your current setup easier, or would like to submit a bug report, we encourage you to open an issue in this repository with the `enhancement` or `bug` labels respectively. If you need help getting started, please ask in the dedicated [#⛏️miners](https://discord.gg/rcgADN9qFX) channel in our Discord. - -### Moving beyond proof of concept - -We provide the MEV-Geth proof of concept as a first milestone on the path to mitigating the negative externalities caused by MEV. We hope to discuss with the community the merits of adopting MEV-Geth in its current form. Our preliminary research indicates it could free at least 2.5% of the current chain congestion by eliminating the use of frontrunning and backrunning and provide uplift of up to 18% on miner rewards from Ethereum. That being said, we believe a sustainable solution to MEV existential risks requires complete privacy and finality, which the proof of concept does not address. We hope to engage community feedback throughout the development of this complete version of MEV-Geth. From 973085d7e10237c8296988b7b194aaa12b523bdf Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Wed, 2 Jun 2021 11:06:51 -0700 Subject: [PATCH 12/17] Add floor gas price for bundle inclusion in merged bundle --- miner/worker.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/miner/worker.go b/miner/worker.go index b4c85cf04376..5d2a87e6f89d 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -1286,8 +1286,12 @@ func (w *worker) mergeBundles(bundles []simulatedBundle, parent *types.Block, he prevState = state.Copy() prevGasPool = new(core.GasPool).AddGas(gasPool.Gas()) + // the floor gas price is 99/100 what was simulated at the top of the block + floorGasPrice := new(big.Int).Mul(bundle.mevGasPrice, big.NewInt(99)) + floorGasPrice = floorGasPrice.Div(floorGasPrice, big.NewInt(100)) + simmed, err := w.computeBundleGas(bundle.originalBundle, parent, header, state, gasPool, pendingTxs, len(finalBundle)) - if err != nil || simmed.totalEth.Cmp(new(big.Int)) <= 0 { + if err != nil || simmed.mevGasPrice.Cmp(floorGasPrice) <= 0 { state = prevState gasPool = prevGasPool continue From 7ffbaac41841d810b7e49114177158e12c3db1c8 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Tue, 15 Jun 2021 10:20:22 -0700 Subject: [PATCH 13/17] flashbots: count eth payments for txs whose nonce is in the mempool Fixes #76 --- miner/worker.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/miner/worker.go b/miner/worker.go index 5d2a87e6f89d..bdc23a084bec 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -1396,16 +1396,16 @@ func (w *worker) computeBundleGas(bundle types.MevBundle, parent *types.Block, h } } + gasUsed := new(big.Int).SetUint64(receipt.GasUsed) + gasFeesTx := gasUsed.Mul(gasUsed, tx.GasPrice()) + coinbaseBalanceAfter := state.GetBalance(w.coinbase) + coinbaseDelta := big.NewInt(0).Sub(coinbaseBalanceAfter, coinbaseBalanceBefore) + coinbaseDelta.Sub(coinbaseDelta, gasFeesTx) + ethSentToCoinbase.Add(ethSentToCoinbase, coinbaseDelta) + if !txInPendingPool { // If tx is not in pending pool, count the gas fees - gasUsed := new(big.Int).SetUint64(receipt.GasUsed) - gasFeesTx := gasUsed.Mul(gasUsed, tx.GasPrice()) gasFees.Add(gasFees, gasFeesTx) - - coinbaseBalanceAfter := state.GetBalance(w.coinbase) - coinbaseDelta := big.NewInt(0).Sub(coinbaseBalanceAfter, coinbaseBalanceBefore) - coinbaseDelta.Sub(coinbaseDelta, gasFeesTx) - ethSentToCoinbase.Add(ethSentToCoinbase, coinbaseDelta) } } From 26055fd02f4ef1009e8df96e2dd13f1c3393b352 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Thu, 27 May 2021 13:47:43 -0700 Subject: [PATCH 14/17] Add flashbots support for eip-1559 --- miner/multi_worker.go | 12 ++++++++++++ miner/worker.go | 31 +++++++++++++++++++++++++++---- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/miner/multi_worker.go b/miner/multi_worker.go index f28fe167db54..b438c14c987c 100644 --- a/miner/multi_worker.go +++ b/miner/multi_worker.go @@ -43,6 +43,18 @@ func (w *multiWorker) isRunning() bool { return false } +// pendingBlockAndReceipts returns pending block and corresponding receipts from the `regularWorker` +func (w *multiWorker) pendingBlockAndReceipts() (*types.Block, types.Receipts) { + // return a snapshot to avoid contention on currentMu mutex + return w.regularWorker.pendingBlockAndReceipts() +} + +func (w *multiWorker) setGasCeil(ceil uint64) { + for _, worker := range w.workers { + worker.setGasCeil(ceil) + } +} + func (w *multiWorker) setExtra(extra []byte) { for _, worker := range w.workers { worker.setExtra(extra) diff --git a/miner/worker.go b/miner/worker.go index bdc23a084bec..33c87417ba19 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -808,6 +808,11 @@ func (w *worker) updateSnapshot() { func (w *worker) commitTransaction(tx *types.Transaction, coinbase common.Address) ([]*types.Log, error) { snap := w.current.state.Snapshot() + gasPrice, err := tx.EffectiveGasTip(w.current.header.BaseFee) + if err != nil { + return nil, err + } + receipt, err := core.ApplyTransaction(w.chainConfig, w.chain, &coinbase, w.current.gasPool, w.current.state, w.current.header, tx, &w.current.header.GasUsed, *w.chain.GetVMConfig()) if err != nil { w.current.state.RevertToSnapshot(snap) @@ -817,7 +822,7 @@ func (w *worker) commitTransaction(tx *types.Transaction, coinbase common.Addres w.current.receipts = append(w.current.receipts, receipt) gasUsed := new(big.Int).SetUint64(receipt.GasUsed) - w.current.profit.Add(w.current.profit, gasUsed.Mul(gasUsed, tx.GasPrice())) + w.current.profit.Add(w.current.profit, gasUsed.Mul(gasUsed, gasPrice)) return receipt.Logs, nil } @@ -874,7 +879,7 @@ func (w *worker) commitBundle(txs types.Transactions, coinbase common.Address, i return true } // Start executing the transaction - w.current.state.Prepare(tx.Hash(), common.Hash{}, w.current.tcount) + w.current.state.Prepare(tx.Hash(), w.current.tcount) logs, err := w.commitTransaction(tx, coinbase) switch { @@ -1365,7 +1370,21 @@ func (w *worker) computeBundleGas(bundle types.MevBundle, parent *types.Block, h ethSentToCoinbase := new(big.Int) for i, tx := range bundle.Txs { - state.Prepare(tx.Hash(), common.Hash{}, i+currentTxCount) + if header.BaseFee != nil && tx.Type() == 2 { + // Sanity check for extremely large numbers + if tx.GasFeeCap().BitLen() > 256 { + return simulatedBundle{}, core.ErrFeeCapVeryHigh + } + if tx.GasTipCap().BitLen() > 256 { + return simulatedBundle{}, core.ErrTipVeryHigh + } + // Ensure gasFeeCap is greater than or equal to gasTipCap. + if tx.GasFeeCapIntCmp(tx.GasTipCap()) < 0 { + return simulatedBundle{}, core.ErrTipAboveFeeCap + } + } + + state.Prepare(tx.Hash(), i+currentTxCount) coinbaseBalanceBefore := state.GetBalance(w.coinbase) receipt, err := core.ApplyTransaction(w.chainConfig, w.chain, &w.coinbase, gasPool, state, header, tx, &tempGasUsed, *w.chain.GetVMConfig()) @@ -1397,7 +1416,11 @@ func (w *worker) computeBundleGas(bundle types.MevBundle, parent *types.Block, h } gasUsed := new(big.Int).SetUint64(receipt.GasUsed) - gasFeesTx := gasUsed.Mul(gasUsed, tx.GasPrice()) + gasPrice, err := tx.EffectiveGasTip(header.BaseFee) + if err != nil { + return simulatedBundle{}, err + } + gasFeesTx := gasUsed.Mul(gasUsed, gasPrice) coinbaseBalanceAfter := state.GetBalance(w.coinbase) coinbaseDelta := big.NewInt(0).Sub(coinbaseBalanceAfter, coinbaseBalanceBefore) coinbaseDelta.Sub(coinbaseDelta, gasFeesTx) From b6fb7b2ce1dcd99bec9ae8472e51e8e7f8e8f7d9 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Wed, 4 Aug 2021 14:10:18 -0700 Subject: [PATCH 15/17] Remove mev specs --- MEV_spec_RPC_v0_1.md | 130 ------------------------------ MEV_spec_RPC_v0_2.md | 133 ------------------------------- MEV_spec_v0_1.md | 120 ---------------------------- MEV_spec_v0_2.md | 184 ------------------------------------------- 4 files changed, 567 deletions(-) delete mode 100644 MEV_spec_RPC_v0_1.md delete mode 100644 MEV_spec_RPC_v0_2.md delete mode 100644 MEV_spec_v0_1.md delete mode 100644 MEV_spec_v0_2.md diff --git a/MEV_spec_RPC_v0_1.md b/MEV_spec_RPC_v0_1.md deleted file mode 100644 index 089b25adbe3d..000000000000 --- a/MEV_spec_RPC_v0_1.md +++ /dev/null @@ -1,130 +0,0 @@ ---- -tags: spec ---- - -# MEV-Geth RPC v0.1 - -# eth_sendBundle - -### Description - -Sends a bundle of transactions to the miner. The bundle has to be executed at the beginning of the block (before any other transactions), with bundle transactions executed exactly in the same order as provided in the bundle. During the Flashbots Alpha this is only called by the Flashbots relay. - -| Name | Type | Description | Comment ---------|----------|-----------|----------- -transactions | `Array` | Array of signed transactions (`eth_sendRawTransaction` style, signed and RLP-encoded) | a no-op in the light mode -blockNumber |`Quantity` |Exact block number at which the bundle can be included. |bundle is evicted after the block -minTimestamp |`Quantity` |Minimum (inclusive) block timestamp at which the bundle can be included. If this value is 0 then any timestamp is acceptable. -maxTimestamp |`Quantity` |Maximum (inclusive) block timestamp at which the bundle can be included. If this value is 0 then any timestamp is acceptable. - -### Returns - -{`boolean`} - `true` if bundle has been accepted by the node, otherwise `false` - -### Example - -```bash -# Request -curl -X POST --data '{ - "id": 1337, - "jsonrpc": "2.0", - "method": "eth_sendBundle", - "params": [ - [ - "f9014946843b9aca00830493e094a011e5f4ea471ee4341a135bb1a4af368155d7a280b8e40d5f2659000000000000000000000000fdd45a22dd1d606b3782f2119621e928e32743000000000000000000000000000000000000000000000000000000000077359400000000000000000000000000000000000000000000000", - "f86e8204d085012a05f200830c350094daf24c20717f428f00d8448d74d67a77f67ceb8287354a6ba7a18000802ea00e411bcb660dd8d47717df89078d2e8160c08e7f11cb7ad0ee935e7436eceb32a013ee00a21b7fa0a9f9c1224d11261648191875d4633aed6003543ea319f12b62" - ], - "0x12ab34", - "0x0", - "0x0" - ] -}' - -# Response -{ - "id": 1337, - "jsonrpc": "2.0", - "result": "true" -} -``` - -# eth_callBundle - -### Description - -Simulate a bundle of transactions at the top of a block. - -After retrieving the block specified in the `blockNrOrHash` it takes the same `blockhash`, `gasLimit`, `difficulty`, same `timestamp` unless the `blockTimestamp` property is specified, and increases the block number by `1`. `eth_callBundle` will timeout after `5` seconds. - -| Name | Type | Description | -| ---- | ---- | ----------- | -| encodedTxs | `Array` | Array of signed transactions (`eth_sendRawTransaction` style, signed and RLP-encoded) | -| blockNrOrHash | `Quantity\|string\|Block Identifier` | Block number, or one of "latest", "earliest" or "pending", or a block identifier as described in {Block Identifier} | -| blockTimestamp |`Quantity` |Block timestamp to be used in replacement of the timestamp taken from the parent block. | - -### Returns - -Map<`Data`, "error|value" : `Data`> - a mapping from transaction hashes to execution results with error or output (value) for each of the transactions - -### Example - -```bash -# Request -curl -X POST --data '{ - "id": 1337, - "jsonrpc": "2.0", - "method": "eth_callBundle", - "params": [ - [ - "f9014946843b9aca00830493e094a011e5f4ea471ee4341a135bb1a4af368155d7a280b8e40d5f2659000000000000000000000000fdd45a22dd1d606b3782f2119621e928e32743000000000000000000000000000000000000000000000000000000000077359400000000000000000000000000000000000000000000000", - "f86e8204d085012a05f200830c350094daf24c20717f428f00d8448d74d67a77f67ceb8287354a6ba7a18000802ea00e411bcb660dd8d47717df89078d2e8160c08e7f11cb7ad0ee935e7436eceb32a013ee00a21b7fa0a9f9c1224d11261648191875d4633aed6003543ea319f12b62" - ], - "0x12ab34" - ] -}' - -# Response -{ - "id": 1337, - "jsonrpc": "2.0", - "result": - { - "0x22b3806fbef9532db4105475222983404783aacd4d865ea5dab76a84aa1a07eb" : { - "value" : "0x0012" - }, - "0x489e3b5493af31d55059f8e296351b267720bc4ba7dc170871c1d789e5541027" : { - "value" : "0xabcd" - } - } -} -``` - ---- - -Below type description can also be found in [EIP-1474](https://eips.ethereum.org/EIPS/eip-1474) - -### `Quantity` - -- A `Quantity` value **MUST** be hex-encoded. -- A `Quantity` value **MUST** be "0x"-prefixed. -- A `Quantity` value **MUST** be expressed using the fewest possible hex digits per byte. -- A `Quantity` value **MUST** express zero as "0x0". - -### `Data` - -- A `Data` value **MUST** be hex-encoded. -- A `Data` value **MUST** be “0x”-prefixed. -- A `Data` value **MUST** be expressed using two hex digits per byte. - -### `Block Identifier` - -Since there is no way to clearly distinguish between a `Data` parameter and a `Quantity` parameter, [EIP-1898](https://eips.ethereum.org/EIPS/eip-1898) provides a format to specify a block either using the block hash or block number. The block identifier is a JSON `object` with the following fields: - -| Position | Name | Type | Description | -| -------- | ---- | ---- | ------------| -| 0A |blockNumber |`Quantity` |The block in the canonical chain with this number | -| 0B |blockHash |`Data` | The block uniquely identified by this hash. The blockNumber and blockHash properties are mutually exclusive; exactly one of them must be set. | -| 1B |requireCanonical |`boolean` | (optional) Whether or not to throw an error if the block is not in the canonical chain as described below. Only allowed in conjunction with the blockHash tag. Defaults to false. | - - -If the block is not found, the callee SHOULD raise a JSON-RPC error (the recommended error code is `-32001: Resource not found`. If the tag is `blockHash` and `requireCanonical` is `true`, the callee SHOULD additionally raise a JSON-RPC error if the block is not in the canonical chain (the recommended error code is `-32000: Invalid input` and in any case should be different than the error code for the block not found case so that the caller can distinguish the cases). The block-not-found check SHOULD take precedence over the block-is-canonical check, so that if the block is not found the callee raises block-not-found rather than block-not-canonical. \ No newline at end of file diff --git a/MEV_spec_RPC_v0_2.md b/MEV_spec_RPC_v0_2.md deleted file mode 100644 index 2bae04de31c8..000000000000 --- a/MEV_spec_RPC_v0_2.md +++ /dev/null @@ -1,133 +0,0 @@ ---- -tags: spec ---- - -# MEV-Geth RPC v0.2 - -# eth_sendBundle - -### Description - -Sends a bundle of transactions to the miner. The bundle has to be executed at the beginning of the block (before any other transactions), with bundle transactions executed exactly in the same order as provided in the bundle. During the Flashbots Alpha this is only called by the Flashbots relay. - -| Name | Type | Description | Comment ---------|----------|-----------|----------- -txs | `Array` | Array of signed transactions (`eth_sendRawTransaction` style, signed and RLP-encoded) | a no-op in the light mode -blockNumber |`Quantity` |Exact block number at which the bundle can be included. |bundle is evicted after the block -minTimestamp |`Quantity` |Minimum (inclusive) block timestamp at which the bundle can be included. If this value is 0 then any timestamp is acceptable. -maxTimestamp |`Quantity` |Maximum (inclusive) block timestamp at which the bundle can be included. If this value is 0 then any timestamp is acceptable. -revertingTxHashes |Array<`Data`> |Array of tx hashes within the bundle that are allowed to cause the EVM execution to revert without preventing the bundle inclusion in a block. - -### Returns - -{`boolean`} - `true` if bundle has been accepted by the node, otherwise `false` - -### Example - -```bash -# Request -curl -X POST --data '{ - "id": 1337, - "jsonrpc": "2.0", - "method": "eth_sendBundle", - "params": [ - { - "txs" : [ - "f9014946843b9aca00830493e094a011e5f4ea471ee4341a135bb1a4af368155d7a280b8e40d5f2659000000000000000000000000fdd45a22dd1d606b3782f2119621e928e32743000000000000000000000000000000000000000000000000000000000077359400000000000000000000000000000000000000000000000", - "f86e8204d085012a05f200830c350094daf24c20717f428f00d8448d74d67a77f67ceb8287354a6ba7a18000802ea00e411bcb660dd8d47717df89078d2e8160c08e7f11cb7ad0ee935e7436eceb32a013ee00a21b7fa0a9f9c1224d11261648191875d4633aed6003543ea319f12b62" - ], - "blockNumber" : "0x12ab34", - "minTimestamp" : "0x0", - "minTimestamp" :"0x0" - } - ] -}' - -# Response -{ - "id": 1337, - "jsonrpc": "2.0", - "result": "true" -} -``` - -# eth_callBundle - -### Description - -Simulate a bundle of transactions at the top of a block. - -After retrieving the block specified in the `blockNrOrHash` it takes the same `blockhash`, `gasLimit`, `difficulty`, same `timestamp` unless the `blockTimestamp` property is specified, and increases the block number by `1`. `eth_callBundle` will timeout after `5` seconds. - -| Name | Type | Description | -| ---- | ---- | ----------- | -| encodedTxs | `Array` | Array of signed transactions (`eth_sendRawTransaction` style, signed and RLP-encoded) | -| blockNrOrHash | `Quantity\|string\|Block Identifier` | Block number, or one of "latest", "earliest" or "pending", or a block identifier as described in {Block Identifier} | -| blockTimestamp |`Quantity` |Block timestamp to be used in replacement of the timestamp taken from the parent block. | - -### Returns - -Map<`Data`, "error|value" : `Data`> - a mapping from transaction hashes to execution results with error or output (value) for each of the transactions - -### Example - -```bash -# Request -curl -X POST --data '{ - "id": 1337, - "jsonrpc": "2.0", - "method": "eth_callBundle", - "params": [ - [ - "f9014946843b9aca00830493e094a011e5f4ea471ee4341a135bb1a4af368155d7a280b8e40d5f2659000000000000000000000000fdd45a22dd1d606b3782f2119621e928e32743000000000000000000000000000000000000000000000000000000000077359400000000000000000000000000000000000000000000000", - "f86e8204d085012a05f200830c350094daf24c20717f428f00d8448d74d67a77f67ceb8287354a6ba7a18000802ea00e411bcb660dd8d47717df89078d2e8160c08e7f11cb7ad0ee935e7436eceb32a013ee00a21b7fa0a9f9c1224d11261648191875d4633aed6003543ea319f12b62" - ], - "0x12ab34" - ] -}' - -# Response -{ - "id": 1337, - "jsonrpc": "2.0", - "result": - { - "0x22b3806fbef9532db4105475222983404783aacd4d865ea5dab76a84aa1a07eb" : { - "value" : "0x0012" - }, - "0x489e3b5493af31d55059f8e296351b267720bc4ba7dc170871c1d789e5541027" : { - "value" : "0xabcd" - } - } -} -``` - ---- - -Below type description can also be found in [EIP-1474](https://eips.ethereum.org/EIPS/eip-1474) - -### `Quantity` - -- A `Quantity` value **MUST** be hex-encoded. -- A `Quantity` value **MUST** be "0x"-prefixed. -- A `Quantity` value **MUST** be expressed using the fewest possible hex digits per byte. -- A `Quantity` value **MUST** express zero as "0x0". - -### `Data` - -- A `Data` value **MUST** be hex-encoded. -- A `Data` value **MUST** be “0x”-prefixed. -- A `Data` value **MUST** be expressed using two hex digits per byte. - -### `Block Identifier` - -Since there is no way to clearly distinguish between a `Data` parameter and a `Quantity` parameter, [EIP-1898](https://eips.ethereum.org/EIPS/eip-1898) provides a format to specify a block either using the block hash or block number. The block identifier is a JSON `object` with the following fields: - -| Position | Name | Type | Description | -| -------- | ---- | ---- | ------------| -| 0A |blockNumber |`Quantity` |The block in the canonical chain with this number | -| 0B |blockHash |`Data` | The block uniquely identified by this hash. The blockNumber and blockHash properties are mutually exclusive; exactly one of them must be set. | -| 1B |requireCanonical |`boolean` | (optional) Whether or not to throw an error if the block is not in the canonical chain as described below. Only allowed in conjunction with the blockHash tag. Defaults to false. | - - -If the block is not found, the callee SHOULD raise a JSON-RPC error (the recommended error code is `-32001: Resource not found`. If the tag is `blockHash` and `requireCanonical` is `true`, the callee SHOULD additionally raise a JSON-RPC error if the block is not in the canonical chain (the recommended error code is `-32000: Invalid input` and in any case should be different than the error code for the block not found case so that the caller can distinguish the cases). The block-not-found check SHOULD take precedence over the block-is-canonical check, so that if the block is not found the callee raises block-not-found rather than block-not-canonical. \ No newline at end of file diff --git a/MEV_spec_v0_1.md b/MEV_spec_v0_1.md deleted file mode 100644 index 0e2a6ebb4f5e..000000000000 --- a/MEV_spec_v0_1.md +++ /dev/null @@ -1,120 +0,0 @@ ---- -tags: spec ---- - -# MEV-Geth v0.1 specification - -## Simple Summary - -Defines the construction and usage of MEV bundles by the miners. Provides specification for custom implementation of required node changes so that MEV bundles can be used correctly. - -## Abstract - -`MevBundles` are stored by the node and the best bundle is added to the block in front of other transactions. `MevBundles` are sorted by their `adjusted gas price`. - -## Motivation - -We believe that without the adoption of neutral, public, open-source infrastructure for permissionless MEV extraction, MEV risks becoming an insiders' game. We commit as an organisation to releasing reference implementations for participation in fair, ethical, and politically neutral MEV extraction. - -## Specification - -The key words `MUST`, `MUST NOT`, `REQUIRED`, `SHALL`, `SHALL NOT`, `SHOULD`, `SHOULD NOT`, `RECOMMENDED`, `MAY`, and `OPTIONAL` in this document are to be interpreted as described in [RFC-2119](https://www.ietf.org/rfc/rfc2119.txt). - -### Definitions - -#### `Bundle` -A set of transactions that `MUST` be executed together and `MUST` be executed at the beginning of the block. - -#### `Unit of work` -A `transaction`, a `bundle` or a `block`. - -#### `Subunit` -A discernible `unit of work` that is a part of a bigger `unit of work`. A `transaction` is a `subunit` of a `bundle` or a `block`. A `bundle` is a `subunit` of a `block`. - -#### `Total gas used` -A sum of gas units used by each transaction from the `unit of work`. - -#### `Average gas price` -Sum of (`gas price` * `total gas used`) of all `subunits` divided by the `total gas used` of the unit. - -#### `Direct coinbase payment` -A value of a transaction with a recipient set to be the same as the `coinbase` address. - -#### `Contract coinbase payment` -A payment from a smart contract to the `coinbase` address. - -#### `Profit` -A difference between the balance of the `coinbase` account at the end and at the beginning of the execution of a `unit of work`. We can measure a `transaction profit`, a `bundle profit`, and a `block profit`. - -Balance of the `coinbase` account changes in the following way -|Unit of work|Balance Change| -|-|-| -|Transaction| `average gas price` * `total gas used` + `direct coinbase payment` + `contract coinbase payment` | -|Bundle | `average gas price` * `total gas used` + `direct coinbase payment` + `contract coinbase payment` | -|Block | `block reward` + `average gas price` * `total gas used` + `direct coinbase payment` + `contract coinbase payment` | - -#### `Adjusted gas price` -`Unit of work` `profit` divided by the `total gas used` by the `unit of work`. - -#### `MevBundle` -An object with four properties: - -|Property| Type|Description| -|-|-|-| -|`transactions`|`Array`|A list of transactions in the bundle. Each transaction is signed and RLP-encoded.| -|`blockNumber`|`uint64`|The exact block number at which the bundle can be executed| -|`minTimestamp`|`uint64`|Minimum block timestamp (inclusive) at which the bundle can be executed| -|`maxTimestamp`|`uint64`|Maximum block timestamp (inclusive) at which the bundle can be executed| - -### Bundle construction - -Bundle `SHOULD` contain transactions with nonces that are following the current nonces of the signing addresses or other transactions preceding them in the same bundle. - -A bundle `MUST` contain at least one transaction. There is no upper limit for the number of transactions in the bundle, however bundles that exceed the block gas limit will always be rejected. - -A bundle `MAY` include a `direct coinbase payment` or a `contract coinbase payment`. Bundles that do not contain such payments may lose comparison when their `profit` is compared with other bundles. - -The `maxTimestamp` value `MUST` be greater or equal the `minTimestamp` value. - -### Accepting bundles from the network - -Node `MUST` provide a way of exposing a JSON RPC endpoint accepting `eth_sendBundle` calls (specified [here](MEV_spec_RPC_v0_1.md)). Such endpoint `SHOULD` only be accepting calls from `MEV-relay` but there is no requirement to restrict it through the node source code as it can be done on the infrastructure level. - -### Bundle eligibility - -Any bundle that is correctly constructed `MUST` have a `blockNumber` field set which specifies in which block it can be included. If the node has already progressed to a later block number then such bundle `MAY` be removed from memory. - -Any bundle that is correctly constructed `MAY` have a `minTimestamp` and/or a `maxTimestamp` field set. Default values for both of these fields are `0` and the meaning of `0` is that any block timestamp value is accepted. When these values are not `0`, then `block.timestamp` is compared with them. If the current `block.timestamp` is greater than the `maxTimestamp` then the bundle `MUST NOT` be included in the block and `MAY` be removed from memory. If the `block.timestamp` is less than `minTimestamp` then the bundle `MUST NOT` be included in the block and `SHOULD NOT` be removed from memory (it awaits future blocks). - -### Block construction - -A block `MUST` either contain one bundle or no bundles. When a bundle is included it `MUST` be the bundle with the highest `adjusted gas price` among eligible bundles. The node `SHOULD` be able to compare a `block profit` in cases when a bundle is included (MEV block) and when no bundles are included (regular block) and choose a block with the highest `profit`. - -A block with a bundle `MUST` place the bundle at the beginning of the block and `MUST NOT` insert any transactions between the bundle transactions. - -### Bundle eviction - -Node `SHOULD` be able to limit the number of bundles kept in memory and apply an algorithm for selecting bundles to be evicted when too many eligible bundles have been received. - -## Rationale - -### At most one MevBundle gets included in the block - -There are two reasons for which multiple bundles in a block may cause problems: - -- two bundles may affect each other's `profit` and so the bundle creator may not be willing to accept a possibility of not being added in the front of the block -- simulating multiple bundle combinations may be very straining for the node infrastructure and introduce excessive latency into the block creation process - -Both of these problems may be addressed in the future versions. - -## Each bundle needs a blockNumber - -This allows specifying bundles to be included in the future blocks (e.g. just after some smart contracts change their state). This cannot be used to ensure a specific parent block / hash. - -## Backwards Compatibility - -This change is not affecting consensus and is fully backwards compatible. - -## Security Considerations - -`MevBundles` that are awaiting future blocks must be stored by the miner's node and it is important to ensure that there is a mechanism to ensure that the storage is limits are not exceeded (whether they are store in memory or persisted). \ No newline at end of file diff --git a/MEV_spec_v0_2.md b/MEV_spec_v0_2.md deleted file mode 100644 index c912b8998520..000000000000 --- a/MEV_spec_v0_2.md +++ /dev/null @@ -1,184 +0,0 @@ ---- -tags: spec ---- - -# MEV-Geth v0.2 specification - -## Simple Summary - -Defines the construction and usage of MEV bundles by miners. Provides a specification for custom implementations of the required node changes so that MEV bundles can be used correctly. - -## Abstract - -`MEV bundles` are stored by the node and the bundles that are providing extra profit for miners are added to the block in front of other transactions. - -## Motivation - -We believe that without the adoption of neutral, public, open-source infrastructure for permissionless MEV extraction, MEV risks becoming an insiders' game. We commit as an organisation to releasing reference implementations for participation in fair, ethical, and politically neutral MEV extraction. - -## Specification - -The key words `MUST`, `MUST NOT`, `REQUIRED`, `SHALL`, `SHALL NOT`, `SHOULD`, `SHOULD NOT`, `RECOMMENDED`, `MAY`, and `OPTIONAL` in this document are to be interpreted as described in [RFC-2119](https://www.ietf.org/rfc/rfc2119.txt). - -### Miner Configuration - -Miner should accept the following configuration options: -* miner.strictprofitswitch (int) - time in miliseconds to wait for both the non-MEV (vanilla) and the `MEV block` construction before selecting the best available block for mining. If value is zero then no waiting is necessary. -* miner.maxmergedbundles (int) - max number of `MEV bundles` to be included within a single block -* relayWSURL (string) - address of the relay WS endpoint -* relayWSSigningKey (bytes32) - a signing key for communication with the relay's WebSockets endpoint. The key should be a valid private key from a secp256k1 based ECDSA. -* relayWSSigningKeystoreFile (string) - path to the keystore file with the WS signing key (used when signing key not provided) -* relayWSKeystorePW (string) - password to unlock the relay WS keystore file - -### Definitions - -#### `Relay` - -An external system delivering `MEV bundles` to the node. Relay provides protection against DoS attacks. - -#### `MEV bundle` or `bundle` - -A list of transactions that `MUST` be executed together and in the same order as provided in the bundle, `MUST` be executed before any non-bundle transactions and only after other bundles that have a higher `bundle adjusted gas price`. - -Transactions in the bundle `MUST` execute without failure (return status 1 on transaction receipts) unless their hashes are included in the `revertingTxHashes` list. - -When representing a bundle in communication between the `relay` and the node we use an object with the following properties: - -|Property| Type|Description| -|-|-|-| -|`txs`|`Array`|A list of transactions in the bundle. Each transaction is signed and RLP-encoded.| -|`blockNumber`|`uint64`|The exact block number at which the bundle can be executed| -|`minTimestamp`|`uint64`|Minimum block timestamp (inclusive) at which the bundle can be executed| -|`maxTimestamp`|`uint64`|Maximum block timestamp (inclusive) at which the bundle can be executed| -|`revertingTxHashes`|`Array`|List of hashes of transactions that are allowed to return status 0 on transaction receipts| - -#### `MEV block` - -A block containing more than zero 'MEV bundles'. - -Whenever we say that a block contains a `bundle` we mean that the block includes all transactions of that bundle in the same order as in the `bundle`. - -#### `Unit of work` -A `transaction`, a `bundle` or a `block`. - -#### `Subunit` -A discernible `unit of work` that is a part of a bigger `unit of work`. A `transaction` is a `subunit` of a `bundle` or a `block`. A `bundle` is a `subunit` of a `block`. - -#### `Total gas used` -The sum of gas units used by each transaction from the `unit of work`. - -#### `Average gas price` -For a transaction it is equivalent to the transaction gas price and for other `units of work` it is a sum of (`average gas price` * `total gas used`) of all `subunits` divided by the `total gas used` by the unit. - -#### `Direct coinbase payment` -The value of a transaction with a recipient set to be the same as the `coinbase` address. - -#### `Contract coinbase payment` -A payment from a smart contract to the `coinbase` address. - -#### `Coinbase payment` -A sum of all `direct coinbase payments` and `contract coinbase payments` within the `unit of work`. - -#### `Eligible coinbase payment` -A sum of all `direct coinbase payments` and `contract coinbase payments` within the `unit of work`. - -#### `Gas fee payment` -An `average gas price` * `total gas used` within the `unit of work`. - -#### `Eligible gas fee payment` -A `gas fee payment` excluding `gas fee payments` from transactions that can be spotted by the miner in the publicly visible transaction pool. - -#### `Bundle scoring profit` -A sum of all `eligible coinbase payments` and `eligible gas payments` of a `bundle`. - -#### `Profit` -A difference between the balance of the `coinbase` account at the end and at the beginning of the execution of a `unit of work`. We can measure a `transaction profit`, a `bundle profit`, and a `block profit`. - -Balance of the `coinbase` account changes in the following way -|Unit of work|Balance Change| -|-|-| -|Transaction| `average gas price` * `total gas used` + `direct coinbase payment` + `contract coinbase payment` | -|Bundle | `average gas price` * `total gas used` + `direct coinbase payment` + `contract coinbase payment` | -|Block | `block reward` + `average gas price` * `total gas used` + `direct coinbase payment` + `contract coinbase payment` | - -#### `Adjusted gas price` -`Unit of work` `profit` divided by the `total gas used` by the `unit of work`. - -#### `Bundle adjusted gas price` -`Bundle scoring profit` divided by the `total gas used` by the `bundle`. - -### Bundle construction - -A bundle `SHOULD` contain transactions with nonces that are following the current nonces of the signing addresses or other transactions preceding them in the same bundle. - -A bundle `MUST` contain at least one transaction. There is no upper limit for the number of transactions in the bundle, however bundles that exceed the block gas limit will always be rejected. - -A bundle `MAY` include `eligible coinbase payments`. Bundles that do not contain such payments may be discarded when their `bundle adjusted gas price` is compared with other bundles. - -The `maxTimestamp` value `MUST` be greater or equal the `minTimestamp` value. - -### Accepting bundles from the network - -#### JSON RPC - -Node `MUST` provide a way of exposing a JSON RPC endpoint accepting `eth_sendBundle` calls (specified [here](MEV_spec_RPC_v0_2.md)). Such endpoint `SHOULD` only be accepting calls from the `relay` but there is no requirement to restrict it through the node source code as it can be done on the infrastructure level. - -#### WebSockets - -Node `MUST` be able to connect to the relay WebSocket endpoint provided as a configuration option named `RelayWSURL` using an authentication key and maintain an open connection. Authentication key `MUST` be provided with the key security in mind. - -During the WebSocket connection initiation and on each reconnection, the node `MUST` sign a timestamp (unix epoch in seconds, UTC) using the `RelayWSSigningKey` and send it in the request header body (X-Auth-Message). The X-Auth-Message should be a JSON encoded object with three properties called timestamp, signature and coinbase. - -### Bundle eligibility - -Any bundle that is correctly constructed `MUST` have a `blockNumber` field set which specifies in which block it can be included. If the node has already progressed to a later block number then such bundle `MAY` be removed from memory. - -Any bundle that is correctly constructed `MAY` have a `minTimestamp` and/or a `maxTimestamp` field set. Default values for both of these fields are `0` and the meaning of `0` is that any block timestamp value is accepted. When these values are not `0`, then `block.timestamp` is compared with them. If the current `block.timestamp` is greater than the `maxTimestamp` then the bundle `MUST NOT` be included in the block and `MAY` be removed from memory. If the `block.timestamp` is less than `minTimestamp` then the bundle `MUST NOT` be included in the block and `SHOULD NOT` be removed from memory (it awaits future blocks). - -### Block construction - -`MEV bundles` `MUST` be sorted by their `bundle adjusted gas price` first and then one by one added to the block as long as there is any gas left in the block and the number of bundles added is less or equal the `MaxMergedBundles` parameter. The remaining block gas `SHOULD` be used for non-MEV transactions. - -Block `MUST` contain between 0 and `MaxMergedBundles` bundles. - -A block with bundles `MUST` place the bundles at the beginning of the block and `MUST NOT` insert any transactions between the bundles or bundle transactions. - -When constructing the block the node should reject any bundle that has a reverting transaction unless its hash is included in the `RevertingTxHashes` list of the bundle object. - -The node `SHOULD` be able to compare the `block profit` for each number of bundles between 0 and `MaxMergedBundles` and choose a block with the highest `profit`, e.g. if `MaxMergedBundles` is 3 then the node `SHOULD` build 4 different blocks - with the maximum of respectively 0, 1, 2, and 3 bundles and choose the one with the highest `profit`. - -When constructing blocks, if the 0 bundles block has not yet been constructed and the `StrictProfitSwitch` parameter is set to a value other than 0, then the node `MUST` wait `StrictProfitSwitch` miliseconds before accepting any MEV block for mining. - -### Bundle eviction - -Node `SHOULD` be able to limit the number of bundles kept in memory and apply an algorithm for selecting bundles to be evicted when too many eligible bundles have been received. - -## Rationale - -### Naive bundle merging - -The bundle merging process is not necessarily picking the most profitable combination of bundles but only the best guess achievable without degrading latency. The first bundle included is always the bundle with the highest `bundle adjusted gas price` - -### Using bundle adjusted gas price instead of adjusted gas price - -The `bundle adjusted gas price` is used to prevent bundle creators from artificially increasing the `adjusted gas price` by adding unrelated high gas price transactions from the publicly visible transaction pool. - -### Each bundle needs a blockNumber - -This allows specifying bundles to be included in the future blocks (e.g. just after some smart contracts change their state). This cannot be used to ensure a specific parent block / hash. - -## Future Considerations - -### Full block submission - -A proposal to allow MEV-Geth accepting fully constructed blocks as well as bundles is considered for inclusion in next versions. - -## Backwards Compatibility - -This change is not affecting consensus and is fully compatible with Ethereum specification. - -Bundle formats are not backwards compatible and the v0.2 bundles would be rejected by v0.1 MEV clients. - -## Security Considerations - -`MEV bundles` that are awaiting future blocks must be stored by the node to ensure that the storage limits are not exceeded (whether they are store in memory or persisted). \ No newline at end of file From 76e3a67994a7ef7cc0f866c0de8fa50dfc96e7b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Kajetan=20Sta=C5=84czak?= Date: Wed, 4 Aug 2021 22:10:54 +0100 Subject: [PATCH 16/17] Update README.md add v0.3 spec link to README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 9cf6a871b9fd..510005c825dc 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ See [here](https://docs.flashbots.net) for Flashbots documentation. | Version | Spec | | ------- | ------------------------------------------------------------------------------------------- | +| v0.3 | [MEV-Geth Spec v0.3](https://docs.flashbots.net/flashbots-auction/miners/mev-geth-spec/v03) | | v0.2 | [MEV-Geth Spec v0.2](https://docs.flashbots.net/flashbots-auction/miners/mev-geth-spec/v02) | | v0.1 | [MEV-Geth Spec v0.1](https://docs.flashbots.net/flashbots-auction/miners/mev-geth-spec/v01) | From 0cdf9ce892d58117cd24476c3666b18de880711d Mon Sep 17 00:00:00 2001 From: eugene Date: Wed, 11 Aug 2021 15:54:46 +0200 Subject: [PATCH 17/17] core: tx_pool not return `error` in `MevBundles()` --- core/tx_pool.go | 4 ++-- core/tx_pool_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/tx_pool.go b/core/tx_pool.go index 2ad073ae3a53..a87f1842cccd 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -550,7 +550,7 @@ func (pool *TxPool) AllMevBundles() []types.MevBundle { // MevBundles returns a list of bundles valid for the given blockNumber/blockTimestamp // also prunes bundles that are outdated -func (pool *TxPool) MevBundles(blockNumber *big.Int, blockTimestamp uint64) ([]types.MevBundle, error) { +func (pool *TxPool) MevBundles(blockNumber *big.Int, blockTimestamp uint64) []types.MevBundle { pool.mu.Lock() defer pool.mu.Unlock() @@ -578,7 +578,7 @@ func (pool *TxPool) MevBundles(blockNumber *big.Int, blockTimestamp uint64) ([]t } pool.mevBundles = bundles - return ret, nil + return ret } // AddMevBundle adds a mev bundle to the pool diff --git a/core/tx_pool_test.go b/core/tx_pool_test.go index 30fc4159a7ac..09908f8639d7 100644 --- a/core/tx_pool_test.go +++ b/core/tx_pool_test.go @@ -2543,7 +2543,7 @@ func BenchmarkInsertRemoteWithAllLocals(b *testing.B) { } func checkBundles(t *testing.T, pool *TxPool, block int64, timestamp uint64, expectedRes int, expectedRemaining int) { - res, _ := pool.MevBundles(big.NewInt(block), timestamp) + res := pool.MevBundles(big.NewInt(block), timestamp) if len(res) != expectedRes { t.Fatalf("expected returned bundles did not match got %d, expected %d", len(res), expectedRes) }