From 930110d77a90694129e39dcefa958e3fae5963b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ljubi=C5=A1a=20Ga=C4=8Devi=C4=87?= <35105035+gacevicljubisa@users.noreply.github.com> Date: Wed, 23 Jul 2025 14:59:43 +0200 Subject: [PATCH 01/96] fix(api): resolve panic when storage incentives disabled (#5150) --- pkg/api/router.go | 23 ++++++++++++++++++----- pkg/api/router_test.go | 26 +++++++++++++------------- 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/pkg/api/router.go b/pkg/api/router.go index 714217c9c11..13a782bcd77 100644 --- a/pkg/api/router.go +++ b/pkg/api/router.go @@ -191,7 +191,7 @@ func (s *Service) checkRouteAvailability(handler http.Handler) http.Handler { func (s *Service) checkSwapAvailability(handler http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if !s.swapEnabled { - jsonhttp.NotImplemented(w, "Swap is disabled. This endpoint is unavailable.") + jsonhttp.Forbidden(w, "Swap is disabled. This endpoint is unavailable.") return } handler.ServeHTTP(w, r) @@ -201,7 +201,17 @@ func (s *Service) checkSwapAvailability(handler http.Handler) http.Handler { func (s *Service) checkChequebookAvailability(handler http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if !s.chequebookEnabled { - jsonhttp.NotImplemented(w, "Chequebook is disabled. This endpoint is unavailable.") + jsonhttp.Forbidden(w, "Chequebook is disabled. This endpoint is unavailable.") + return + } + handler.ServeHTTP(w, r) + }) +} + +func (s *Service) checkStorageIncentivesAvailability(handler http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if s.redistributionAgent == nil { + jsonhttp.Forbidden(w, "Storage incentives are disabled. This endpoint is unavailable.") return } handler.ServeHTTP(w, r) @@ -635,9 +645,12 @@ func (s *Service) mountBusinessDebug() { })), ) - handle("/redistributionstate", jsonhttp.MethodHandler{ - "GET": http.HandlerFunc(s.redistributionStatusHandler), - }) + handle("/redistributionstate", web.ChainHandlers( + s.checkStorageIncentivesAvailability, + web.FinalHandler(jsonhttp.MethodHandler{ + "GET": http.HandlerFunc(s.redistributionStatusHandler), + })), + ) handle("/status", jsonhttp.MethodHandler{ "GET": web.ChainHandlers( diff --git a/pkg/api/router_test.go b/pkg/api/router_test.go index 22ba15c8cde..e4cabc8a4c1 100644 --- a/pkg/api/router_test.go +++ b/pkg/api/router_test.go @@ -281,17 +281,17 @@ func TestEndpointOptions(t *testing.T) { {"/consumed", []string{"GET"}, http.StatusNoContent}, {"/consumed/{peer}", []string{"GET"}, http.StatusNoContent}, {"/timesettlements", []string{"GET"}, http.StatusNoContent}, - {"/settlements", nil, http.StatusNotImplemented}, - {"/settlements/{peer}", nil, http.StatusNotImplemented}, - {"/chequebook/cheque/{peer}", nil, http.StatusNotImplemented}, - {"/chequebook/cheque", nil, http.StatusNotImplemented}, - {"/chequebook/cashout/{peer}", nil, http.StatusNotImplemented}, + {"/settlements", nil, http.StatusForbidden}, + {"/settlements/{peer}", nil, http.StatusForbidden}, + {"/chequebook/cheque/{peer}", nil, http.StatusForbidden}, + {"/chequebook/cheque", nil, http.StatusForbidden}, + {"/chequebook/cashout/{peer}", nil, http.StatusForbidden}, {"/chequebook/balance", []string{"GET"}, http.StatusNoContent}, {"/chequebook/address", []string{"GET"}, http.StatusNoContent}, {"/chequebook/deposit", []string{"POST"}, http.StatusNoContent}, {"/chequebook/withdraw", []string{"POST"}, http.StatusNoContent}, - {"/wallet", nil, http.StatusNotImplemented}, - {"/wallet/withdraw/{coin}", nil, http.StatusNotImplemented}, + {"/wallet", nil, http.StatusForbidden}, + {"/wallet/withdraw/{coin}", nil, http.StatusForbidden}, {"/stamps", []string{"GET"}, http.StatusNoContent}, {"/stamps/{batch_id}", []string{"GET"}, http.StatusNoContent}, {"/stamps/{batch_id}/buckets", []string{"GET"}, http.StatusNoContent}, @@ -381,12 +381,12 @@ func TestEndpointOptions(t *testing.T) { {"/chequebook/cheque/{peer}", []string{"GET"}, http.StatusNoContent}, {"/chequebook/cheque", []string{"GET"}, http.StatusNoContent}, {"/chequebook/cashout/{peer}", []string{"GET", "POST"}, http.StatusNoContent}, - {"/chequebook/balance", nil, http.StatusNotImplemented}, - {"/chequebook/address", nil, http.StatusNotImplemented}, - {"/chequebook/deposit", nil, http.StatusNotImplemented}, - {"/chequebook/withdraw", nil, http.StatusNotImplemented}, - {"/wallet", nil, http.StatusNotImplemented}, - {"/wallet/withdraw/{coin}", nil, http.StatusNotImplemented}, + {"/chequebook/balance", nil, http.StatusForbidden}, + {"/chequebook/address", nil, http.StatusForbidden}, + {"/chequebook/deposit", nil, http.StatusForbidden}, + {"/chequebook/withdraw", nil, http.StatusForbidden}, + {"/wallet", nil, http.StatusForbidden}, + {"/wallet/withdraw/{coin}", nil, http.StatusForbidden}, {"/stamps", []string{"GET"}, http.StatusNoContent}, {"/stamps/{batch_id}", []string{"GET"}, http.StatusNoContent}, {"/stamps/{batch_id}/buckets", []string{"GET"}, http.StatusNoContent}, From 1fc45e9c97bea56bac77a57db12641f86050fcfd Mon Sep 17 00:00:00 2001 From: nugaon Date: Thu, 24 Jul 2025 13:02:57 +0200 Subject: [PATCH 02/96] fix: remove setting peer healthy true in case of underpopulated bins (#5145) --- pkg/node/node.go | 2 +- pkg/salud/salud.go | 22 +++++----------------- pkg/salud/salud_test.go | 10 +++++----- 3 files changed, 11 insertions(+), 23 deletions(-) diff --git a/pkg/node/node.go b/pkg/node/node.go index 7d14ceb1b15..76e4c6a2128 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -975,7 +975,7 @@ func NewBee( return nil, fmt.Errorf("status service: %w", err) } - saludService := salud.New(nodeStatus, kad, localStore, logger, detector, api.FullMode.String(), salud.DefaultMinPeersPerBin, salud.DefaultDurPercentile, salud.DefaultConnsPercentile) + saludService := salud.New(nodeStatus, kad, localStore, logger, detector, api.FullMode.String(), salud.DefaultDurPercentile, salud.DefaultConnsPercentile) b.saludCloser = saludService rC, unsub := saludService.SubscribeNetworkStorageRadius() diff --git a/pkg/salud/salud.go b/pkg/salud/salud.go index 8a9937adee2..dc931f86081 100644 --- a/pkg/salud/salud.go +++ b/pkg/salud/salud.go @@ -29,7 +29,6 @@ const ( initialBackoffDelay = 10 * time.Second maxBackoffDelay = 5 * time.Minute backoffFactor = 2 - DefaultMinPeersPerBin = 4 DefaultDurPercentile = 0.4 // consider 40% as healthy, lower percentile = stricter duration check DefaultConnsPercentile = 0.8 // consider 80% as healthy, lower percentile = stricter conns check ) @@ -64,7 +63,6 @@ func New( logger log.Logger, startupStabilizer stabilization.Subscriber, mode string, - minPeersPerbin int, durPercentile float64, connsPercentile float64, ) *service { @@ -81,12 +79,12 @@ func New( } s.wg.Add(1) - go s.worker(startupStabilizer, mode, minPeersPerbin, durPercentile, connsPercentile) + go s.worker(startupStabilizer, mode, durPercentile, connsPercentile) return s } -func (s *service) worker(startupStabilizer stabilization.Subscriber, mode string, minPeersPerbin int, durPercentile float64, connsPercentile float64) { +func (s *service) worker(startupStabilizer stabilization.Subscriber, mode string, durPercentile float64, connsPercentile float64) { defer s.wg.Done() sub, unsubscribe := startupStabilizer.Subscribe() @@ -102,7 +100,7 @@ func (s *service) worker(startupStabilizer stabilization.Subscriber, mode string currentDelay := initialBackoffDelay for { - s.salud(mode, minPeersPerbin, durPercentile, connsPercentile) + s.salud(mode, durPercentile, connsPercentile) select { case <-s.quit: @@ -134,13 +132,12 @@ type peer struct { // salud acquires the status snapshot of every peer and computes an nth percentile of response duration and connected // per count, the most common storage radius, and the batch commitment, and based on these values, marks peers as unhealhy that fall beyond // the allowed thresholds. -func (s *service) salud(mode string, minPeersPerbin int, durPercentile float64, connsPercentile float64) { +func (s *service) salud(mode string, durPercentile float64, connsPercentile float64) { var ( mtx sync.Mutex wg sync.WaitGroup totaldur float64 peers []peer - bins [swarm.MaxBins]int ) err := s.topology.EachConnectedPeer(func(addr swarm.Address, bin uint8) (stop bool, jumpToNext bool, err error) { @@ -165,7 +162,6 @@ func (s *service) salud(mode string, minPeersPerbin int, durPercentile float64, } mtx.Lock() - bins[bin]++ totaldur += dur.Seconds() peers = append(peers, peer{snapshot, dur, addr, bin, s.reserve.IsWithinStorageRadius(addr)}) mtx.Unlock() @@ -206,17 +202,10 @@ func (s *service) salud(mode string, minPeersPerbin int, durPercentile float64, var healthy bool - // every bin should have at least some peers, healthy or not - if bins[peer.bin] <= minPeersPerbin { - s.metrics.Healthy.Inc() - s.topology.UpdatePeerHealth(peer.addr, true, peer.dur) - continue - } - if networkRadius > 0 && peer.status.CommittedDepth < uint32(networkRadius-2) { s.logger.Debug("radius health failure", "radius", peer.status.CommittedDepth, "peer_address", peer.addr, "bin", peer.bin) } else if peer.dur.Seconds() > pDur { - s.logger.Debug("response duration below threshold", "duration", peer.dur, "peer_address", peer.addr, "bin", peer.bin) + s.logger.Debug("response duration above threshold", "duration", peer.dur, "peer_address", peer.addr, "bin", peer.bin) } else if peer.status.ConnectedPeers < pConns { s.logger.Debug("connections count below threshold", "connections", peer.status.ConnectedPeers, "peer_address", peer.addr, "bin", peer.bin) } else if peer.status.BatchCommitment != commitment { @@ -230,7 +219,6 @@ func (s *service) salud(mode string, minPeersPerbin int, durPercentile float64, s.metrics.Healthy.Inc() } else { s.metrics.Unhealthy.Inc() - bins[peer.bin]-- } } diff --git a/pkg/salud/salud_test.go b/pkg/salud/salud_test.go index 3f40d54ad2e..4123702263e 100644 --- a/pkg/salud/salud_test.go +++ b/pkg/salud/salud_test.go @@ -72,7 +72,7 @@ func TestSalud(t *testing.T) { mockstorer.WithCapacityDoubling(2), ) - service := salud.New(statusM, topM, reserve, log.Noop, stabilmock.NewSubscriber(true), "full", 0, 0.8, 0.8) + service := salud.New(statusM, topM, reserve, log.Noop, stabilmock.NewSubscriber(true), "full", 0.8, 0.8) err := spinlock.Wait(time.Minute, func() bool { return len(topM.PeersHealth()) == len(peers) @@ -119,7 +119,7 @@ func TestSelfUnhealthyRadius(t *testing.T) { mockstorer.WithCapacityDoubling(0), ) - service := salud.New(statusM, topM, reserve, log.Noop, stabilmock.NewSubscriber(true), "full", 0, 0.8, 0.8) + service := salud.New(statusM, topM, reserve, log.Noop, stabilmock.NewSubscriber(true), "full", 0.8, 0.8) testutil.CleanupCloser(t, service) err := spinlock.Wait(time.Minute, func() bool { @@ -157,7 +157,7 @@ func TestSelfHealthyCapacityDoubling(t *testing.T) { mockstorer.WithCapacityDoubling(2), ) - service := salud.New(statusM, topM, reserve, log.Noop, stabilmock.NewSubscriber(true), "full", 0, 0.8, 0.8) + service := salud.New(statusM, topM, reserve, log.Noop, stabilmock.NewSubscriber(true), "full", 0.8, 0.8) testutil.CleanupCloser(t, service) err := spinlock.Wait(time.Minute, func() bool { @@ -187,7 +187,7 @@ func TestSubToRadius(t *testing.T) { topM := topMock.NewTopologyDriver(topMock.WithPeers(addrs...)) - service := salud.New(&statusMock{make(map[string]peer)}, topM, mockstorer.NewReserve(), log.Noop, stabilmock.NewSubscriber(true), "full", 0, 0.8, 0.8) + service := salud.New(&statusMock{make(map[string]peer)}, topM, mockstorer.NewReserve(), log.Noop, stabilmock.NewSubscriber(true), "full", 0.8, 0.8) c, unsub := service.SubscribeNetworkStorageRadius() t.Cleanup(unsub) @@ -220,7 +220,7 @@ func TestUnsub(t *testing.T) { topM := topMock.NewTopologyDriver(topMock.WithPeers(addrs...)) - service := salud.New(&statusMock{make(map[string]peer)}, topM, mockstorer.NewReserve(), log.Noop, stabilmock.NewSubscriber(true), "full", 0, 0.8, 0.8) + service := salud.New(&statusMock{make(map[string]peer)}, topM, mockstorer.NewReserve(), log.Noop, stabilmock.NewSubscriber(true), "full", 0.8, 0.8) testutil.CleanupCloser(t, service) c, unsub := service.SubscribeNetworkStorageRadius() From a3d30d0da01a834586b31b2a4fa75cdbbb2ff11e Mon Sep 17 00:00:00 2001 From: Calin M Date: Fri, 25 Jul 2025 09:45:45 +0300 Subject: [PATCH 03/96] fix: do not import postage snapshot on ultra light nodes (#5169) --- pkg/node/node.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/node/node.go b/pkg/node/node.go index 76e4c6a2128..c828c342c98 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -803,7 +803,7 @@ func NewBee( } ) - if !o.SkipPostageSnapshot && !batchStoreExists && (networkID == mainnetNetworkID) { + if !o.SkipPostageSnapshot && !batchStoreExists && (networkID == mainnetNetworkID) && beeNodeMode != api.UltraLightMode { chainBackend := NewSnapshotLogFilterer(logger, archiveSnapshotGetter{}) snapshotEventListener := listener.New(b.syncingStopped, logger, chainBackend, postageStampContractAddress, postageStampContractABI, o.BlockTime, postageSyncingStallingTimeout, postageSyncingBackoffTimeout) From 4dc7507cf35a6af7b7b5ca5cd29a12de0e001bec Mon Sep 17 00:00:00 2001 From: jishudashu <979260390@qq.com> Date: Mon, 28 Jul 2025 15:18:38 +0800 Subject: [PATCH 04/96] refactor: use maps.Copy for cleaner map handling (#5168) Signed-off-by: jishudashu <979260390@qq.com> --- pkg/accounting/accounting.go | 5 ++--- pkg/transaction/backendsimulation/backend.go | 9 +++------ 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/pkg/accounting/accounting.go b/pkg/accounting/accounting.go index 35316e3a73b..45e4ab14f79 100644 --- a/pkg/accounting/accounting.go +++ b/pkg/accounting/accounting.go @@ -10,6 +10,7 @@ import ( "context" "errors" "fmt" + "maps" "math/big" "strings" "sync" @@ -721,9 +722,7 @@ func (a *Accounting) PeerAccounting() (map[string]PeerInfo, error) { a.accountingPeersMu.Lock() accountingPeersList := make(map[string]*accountingPeer) - for peer, accountingPeer := range a.accountingPeers { - accountingPeersList[peer] = accountingPeer - } + maps.Copy(accountingPeersList, a.accountingPeers) a.accountingPeersMu.Unlock() for peer, accountingPeer := range accountingPeersList { diff --git a/pkg/transaction/backendsimulation/backend.go b/pkg/transaction/backendsimulation/backend.go index 58c183fa0bd..f552b47b9bf 100644 --- a/pkg/transaction/backendsimulation/backend.go +++ b/pkg/transaction/backendsimulation/backend.go @@ -7,6 +7,7 @@ package backendsimulation import ( "context" "errors" + "maps" "math/big" "github.com/ethereum/go-ethereum" @@ -74,15 +75,11 @@ func (m *simulatedBackend) advanceBlock() { m.blockNumber = block.Number if block.Receipts != nil { - for hash, receipt := range block.Receipts { - m.receipts[hash] = receipt - } + maps.Copy(m.receipts, block.Receipts) } if block.NoncesAt != nil { - for addr, nonce := range block.NoncesAt { - m.noncesAt[addr] = nonce - } + maps.Copy(m.noncesAt, block.NoncesAt) } } From 708d869427071044090ea546645e3a56b3f88a44 Mon Sep 17 00:00:00 2001 From: Not Darko <93942788+darkobas2@users.noreply.github.com> Date: Mon, 28 Jul 2025 12:01:44 +0200 Subject: [PATCH 05/96] fix: [goreleaser]skip rc upload for brews (#5031) --- .goreleaser.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.goreleaser.yml b/.goreleaser.yml index 8934e0fde66..78066917a98 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -293,6 +293,14 @@ brews: email: bee-worker@ethswarm.org homepage: https://swarm.ethereum.org/ description: Ethereum Swarm node + # Setting this will prevent goreleaser to actually try to commit the updated + # formula - instead, the formula file will be stored on the dist directory + # only, leaving the responsibility of publishing it to the user. + # If set to auto, the release will not be uploaded to the homebrew tap + # in case there is an indicator for prerelease in the tag e.g. v1.0.0-rc1 + # + # Templates: allowed. + skip_upload: auto caveats: | Logs: #{var}/log/swarm-bee/bee.log Config: #{etc}/swarm-bee/bee.yaml From 866b4b73c64d7f2fb6cb4c9184448768a249f777 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ljubi=C5=A1a=20Ga=C4=8Devi=C4=87?= <35105035+gacevicljubisa@users.noreply.github.com> Date: Tue, 29 Jul 2025 13:47:31 +0200 Subject: [PATCH 06/96] feat(storer): use buf chan to improve perfomance (#5170) --- pkg/storer/sample.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pkg/storer/sample.go b/pkg/storer/sample.go index e02027ce00b..c2b17caecab 100644 --- a/pkg/storer/sample.go +++ b/pkg/storer/sample.go @@ -65,7 +65,6 @@ func (db *DB) ReserveSample( consensusTime uint64, minBatchBalance *big.Int, ) (Sample, error) { - g, ctx := errgroup.WithContext(ctx) allStats := &SampleStats{} @@ -85,7 +84,8 @@ func (db *DB) ReserveSample( allStats.BatchesBelowValueDuration = time.Since(t) - chunkC := make(chan *reserve.ChunkBinItem) + workers := max(4, runtime.NumCPU()) + chunkC := make(chan *reserve.ChunkBinItem, 3*workers) // Phase 1: Iterate chunk addresses g.Go(func() error { @@ -113,16 +113,15 @@ func (db *DB) ReserveSample( }) // Phase 2: Get the chunk data and calculate transformed hash - sampleItemChan := make(chan SampleItem) + sampleItemChan := make(chan SampleItem, 3*workers) prefixHasherFactory := func() hash.Hash { return swarm.NewPrefixHasher(anchor) } - workers := max(4, runtime.NumCPU()) db.logger.Debug("reserve sampler workers", "count", workers) - for i := 0; i < workers; i++ { + for range workers { g.Go(func() error { wstat := SampleStats{} hasher := bmt.NewHasher(prefixHasherFactory) From a12e6f22a0d4b966621d5b963057fc68f7d4dd08 Mon Sep 17 00:00:00 2001 From: Acha Bill <57879913+acha-bill@users.noreply.github.com> Date: Tue, 29 Jul 2025 07:48:21 -0400 Subject: [PATCH 07/96] feat: add reserveCapacityDoubling to /reservestate (#5139) --- openapi/SwarmCommon.yaml | 2 ++ pkg/api/postage.go | 14 ++++++++------ pkg/api/postage_test.go | 9 +++++++-- pkg/storer/mock/mockreserve.go | 6 ++++++ pkg/storer/mock/mockstorer.go | 23 +++++++++++++++++++++-- pkg/storer/reserve.go | 7 +++++++ pkg/storer/storer.go | 1 + 7 files changed, 52 insertions(+), 10 deletions(-) diff --git a/openapi/SwarmCommon.yaml b/openapi/SwarmCommon.yaml index 9fa0bdb7528..082b8494fdb 100644 --- a/openapi/SwarmCommon.yaml +++ b/openapi/SwarmCommon.yaml @@ -50,6 +50,8 @@ components: type: integer commitment: type: integer + reserveCapacityDoubling: + type: integer ChainState: type: object diff --git a/pkg/api/postage.go b/pkg/api/postage.go index 9d6aa2bcc11..d9de74ca354 100644 --- a/pkg/api/postage.go +++ b/pkg/api/postage.go @@ -366,9 +366,10 @@ func (s *Service) postageGetStampHandler(w http.ResponseWriter, r *http.Request) } type reserveStateResponse struct { - Radius uint8 `json:"radius"` - StorageRadius uint8 `json:"storageRadius"` - Commitment uint64 `json:"commitment"` + Radius uint8 `json:"radius"` + StorageRadius uint8 `json:"storageRadius"` + Commitment uint64 `json:"commitment"` + ReserveCapacityDoubling uint8 `json:"reserveCapacityDoubling"` } type chainStateResponse struct { @@ -390,9 +391,10 @@ func (s *Service) reserveStateHandler(w http.ResponseWriter, _ *http.Request) { } jsonhttp.OK(w, reserveStateResponse{ - Radius: s.batchStore.Radius(), - StorageRadius: s.storer.StorageRadius(), - Commitment: commitment, + Radius: s.batchStore.Radius(), + StorageRadius: s.storer.StorageRadius(), + Commitment: commitment, + ReserveCapacityDoubling: s.storer.CapacityDoubling(), }) } diff --git a/pkg/api/postage_test.go b/pkg/api/postage_test.go index 6ba9c812a69..bc476be99da 100644 --- a/pkg/api/postage_test.go +++ b/pkg/api/postage_test.go @@ -374,13 +374,18 @@ func TestReserveState(t *testing.T) { t.Run("ok", func(t *testing.T) { t.Parallel() + s := mockstorer.New() + s.SetStorageRadius(3) + s.SetCommittedDepth(5) ts, _, _, _ := newTestServer(t, testServerOptions{ BatchStore: mock.New(mock.WithRadius(5)), - Storer: mockstorer.New(), + Storer: s, }) jsonhttptest.Request(t, ts, http.MethodGet, "/reservestate", http.StatusOK, jsonhttptest.WithExpectedJSONResponse(&api.ReserveStateResponse{ - Radius: 5, + Radius: 5, + StorageRadius: 3, + ReserveCapacityDoubling: 2, }), ) }) diff --git a/pkg/storer/mock/mockreserve.go b/pkg/storer/mock/mockreserve.go index ba8f13590ef..7ac74ee49ce 100644 --- a/pkg/storer/mock/mockreserve.go +++ b/pkg/storer/mock/mockreserve.go @@ -152,6 +152,12 @@ func (s *ReserveStore) CommittedDepth() uint8 { return s.radius + uint8(s.capacityDoubling) } +func (s *ReserveStore) CapacityDoubling() uint8 { + s.mtx.Lock() + defer s.mtx.Unlock() + return uint8(s.capacityDoubling) +} + // IntervalChunks returns a set of chunk in a requested interval. func (s *ReserveStore) SubscribeBin(ctx context.Context, bin uint8, start uint64) (<-chan *storer.BinC, func(), <-chan error) { s.mtx.Lock() diff --git a/pkg/storer/mock/mockstorer.go b/pkg/storer/mock/mockstorer.go index d0b5b7e6ad0..efdce20bbf4 100644 --- a/pkg/storer/mock/mockstorer.go +++ b/pkg/storer/mock/mockstorer.go @@ -28,6 +28,9 @@ type mockStorer struct { activeSessions map[uint64]*storer.SessionInfo chunkPushC chan *pusher.Op debugInfo storer.Info + + storageRadius uint8 + committedDepth uint8 } type putterSession struct { @@ -218,9 +221,13 @@ func (m *mockStorer) ChunkStore() storage.ReadOnlyChunkStore { return m.chunkStore } -func (m *mockStorer) StorageRadius() uint8 { return 0 } +func (m *mockStorer) StorageRadius() uint8 { return m.storageRadius } + +func (m *mockStorer) CommittedDepth() uint8 { return m.committedDepth } -func (m *mockStorer) CommittedDepth() uint8 { return 0 } +func (m *mockStorer) CapacityDoubling() uint8 { + return m.committedDepth - m.storageRadius +} func (m *mockStorer) IsWithinStorageRadius(_ swarm.Address) bool { return true } @@ -235,3 +242,15 @@ func (m *mockStorer) NeighborhoodsStat(ctx context.Context) ([]*storer.Neighborh func (m *mockStorer) Put(ctx context.Context, ch swarm.Chunk) error { return m.chunkStore.Put(ctx, ch) } + +func (m *mockStorer) SetStorageRadius(radius uint8) { + m.mu.Lock() + defer m.mu.Unlock() + m.storageRadius = radius +} + +func (m *mockStorer) SetCommittedDepth(depth uint8) { + m.mu.Lock() + defer m.mu.Unlock() + m.committedDepth = depth +} diff --git a/pkg/storer/reserve.go b/pkg/storer/reserve.go index 329e9461f50..c79b2b9fb06 100644 --- a/pkg/storer/reserve.go +++ b/pkg/storer/reserve.go @@ -428,6 +428,13 @@ func (db *DB) CommittedDepth() uint8 { return uint8(db.reserveOptions.capacityDoubling) + db.reserve.Radius() } +func (db *DB) CapacityDoubling() uint8 { + if db.reserve == nil { + return 0 + } + return uint8(db.reserveOptions.capacityDoubling) +} + func (db *DB) ReserveSize() int { if db.reserve == nil { return 0 diff --git a/pkg/storer/storer.go b/pkg/storer/storer.go index cafcb3c6a57..76b35f198ba 100644 --- a/pkg/storer/storer.go +++ b/pkg/storer/storer.go @@ -165,6 +165,7 @@ type RadiusChecker interface { IsWithinStorageRadius(addr swarm.Address) bool StorageRadius() uint8 CommittedDepth() uint8 + CapacityDoubling() uint8 } // LocalStore is a read-only ChunkStore. It can be used to check if chunk is known From 15a8d026c281242ef8f7157137c76caf92516b4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ljubi=C5=A1a=20Ga=C4=8Devi=C4=87?= <35105035+gacevicljubisa@users.noreply.github.com> Date: Tue, 29 Jul 2025 15:22:54 +0200 Subject: [PATCH 08/96] fix: replace SuggestGasPrice with custom SuggestedFeeAndTip (#5144) --- cmd/bee/cmd/cmd.go | 2 + cmd/bee/cmd/deploy.go | 1 + cmd/bee/cmd/start.go | 1 + packaging/bee.yaml | 2 + packaging/homebrew-amd64/bee.yaml | 2 + packaging/homebrew-arm64/bee.yaml | 2 + packaging/scoop/bee.yaml | 2 + pkg/api/redistribution_test.go | 4 +- pkg/node/chain.go | 23 +-- pkg/node/node.go | 14 +- pkg/settlement/swap/chequebook/init.go | 4 +- pkg/storageincentives/agent.go | 4 +- pkg/storageincentives/agent_test.go | 4 +- .../redistribution/redistribution.go | 11 +- pkg/transaction/backend.go | 26 +-- pkg/transaction/backend/backend.go | 32 +++ pkg/transaction/backend_test.go | 22 +-- pkg/transaction/backendmock/backend.go | 54 +---- pkg/transaction/backendsimulation/backend.go | 24 +-- pkg/transaction/transaction.go | 62 +----- pkg/transaction/transaction_test.go | 184 ++++++------------ pkg/transaction/wrapped/fee.go | 61 ++++++ pkg/transaction/wrapped/fee_test.go | 128 ++++++++++++ pkg/transaction/wrapped/metrics.go | 22 +-- pkg/transaction/wrapped/wrapped.go | 59 ++---- 25 files changed, 365 insertions(+), 385 deletions(-) create mode 100644 pkg/transaction/backend/backend.go create mode 100644 pkg/transaction/wrapped/fee.go create mode 100644 pkg/transaction/wrapped/fee_test.go diff --git a/cmd/bee/cmd/cmd.go b/cmd/bee/cmd/cmd.go index e10e015ce81..5b2c156c630 100644 --- a/cmd/bee/cmd/cmd.go +++ b/cmd/bee/cmd/cmd.go @@ -81,6 +81,7 @@ const ( optionMinimumStorageRadius = "minimum-storage-radius" optionReserveCapacityDoubling = "reserve-capacity-doubling" optionSkipPostageSnapshot = "skip-postage-snapshot" + optionNameMinimumGasTipCap = "minimum-gas-tip-cap" ) // nolint:gochecknoinits @@ -290,6 +291,7 @@ func (c *command) setAllFlags(cmd *cobra.Command) { cmd.Flags().Uint(optionMinimumStorageRadius, 0, "minimum radius storage threshold") cmd.Flags().Int(optionReserveCapacityDoubling, 0, "reserve capacity doubling") cmd.Flags().Bool(optionSkipPostageSnapshot, false, "skip postage snapshot") + cmd.Flags().Uint64(optionNameMinimumGasTipCap, 0, "minimum gas tip cap in wei for transactions, 0 means use suggested gas tip cap") } func newLogger(cmd *cobra.Command, verbosity string) (log.Logger, error) { diff --git a/cmd/bee/cmd/deploy.go b/cmd/bee/cmd/deploy.go index 89880273dab..c0e20cb63e7 100644 --- a/cmd/bee/cmd/deploy.go +++ b/cmd/bee/cmd/deploy.go @@ -59,6 +59,7 @@ func (c *command) initDeployCmd() error { signer, blocktime, true, + c.config.GetUint64(optionNameMinimumGasTipCap), ) if err != nil { return err diff --git a/cmd/bee/cmd/start.go b/cmd/bee/cmd/start.go index dab68c447fb..c6662bcba57 100644 --- a/cmd/bee/cmd/start.go +++ b/cmd/bee/cmd/start.go @@ -299,6 +299,7 @@ func buildBeeNode(ctx context.Context, c *command, cmd *cobra.Command, logger lo EnableWS: c.config.GetBool(optionNameP2PWSEnable), FullNodeMode: fullNode, Logger: logger, + MinimumGasTipCap: c.config.GetUint64(optionNameMinimumGasTipCap), MinimumStorageRadius: c.config.GetUint(optionMinimumStorageRadius), MutexProfile: c.config.GetBool(optionNamePProfMutex), NATAddr: c.config.GetString(optionNameNATAddr), diff --git a/packaging/bee.yaml b/packaging/bee.yaml index 8176dfb47df..42ee156d0de 100644 --- a/packaging/bee.yaml +++ b/packaging/bee.yaml @@ -38,6 +38,8 @@ data-dir: "/var/lib/bee" # help: false ## triggers connect to main net bootnodes. # mainnet: true +## minimum gas tip cap in wei for transactions, 0 means use suggested gas tip cap +# minimum-gas-tip-cap: 0 ## minimum radius storage threshold # minimum-storage-radius: "0" ## NAT exposed address diff --git a/packaging/homebrew-amd64/bee.yaml b/packaging/homebrew-amd64/bee.yaml index c771e5d72c2..14a32f70116 100644 --- a/packaging/homebrew-amd64/bee.yaml +++ b/packaging/homebrew-amd64/bee.yaml @@ -38,6 +38,8 @@ data-dir: "/usr/local/var/lib/swarm-bee" # help: false ## triggers connect to main net bootnodes. # mainnet: true +## minimum gas tip cap in wei for transactions, 0 means use suggested gas tip cap +# minimum-gas-tip-cap: 0 ## minimum radius storage threshold # minimum-storage-radius: "0" ## NAT exposed address diff --git a/packaging/homebrew-arm64/bee.yaml b/packaging/homebrew-arm64/bee.yaml index 057d6c54d32..55d522b601c 100644 --- a/packaging/homebrew-arm64/bee.yaml +++ b/packaging/homebrew-arm64/bee.yaml @@ -38,6 +38,8 @@ data-dir: "/opt/homebrew/var/lib/swarm-bee" # help: false ## triggers connect to main net bootnodes. # mainnet: true +## minimum gas tip cap in wei for transactions, 0 means use suggested gas tip cap +# minimum-gas-tip-cap: 0 ## minimum radius storage threshold # minimum-storage-radius: "0" ## NAT exposed address diff --git a/packaging/scoop/bee.yaml b/packaging/scoop/bee.yaml index d2f97fedd33..70bed38f947 100644 --- a/packaging/scoop/bee.yaml +++ b/packaging/scoop/bee.yaml @@ -38,6 +38,8 @@ data-dir: "./data" # help: false ## triggers connect to main net bootnodes. # mainnet: true +## minimum gas tip cap in wei for transactions, 0 means use suggested gas tip cap +# minimum-gas-tip-cap: 0 ## minimum radius storage threshold # minimum-storage-radius: "0" ## NAT exposed address diff --git a/pkg/api/redistribution_test.go b/pkg/api/redistribution_test.go index 07b4bc28cfe..eef01977866 100644 --- a/pkg/api/redistribution_test.go +++ b/pkg/api/redistribution_test.go @@ -46,8 +46,8 @@ func TestRedistributionStatus(t *testing.T) { backendmock.WithBalanceAt(func(ctx context.Context, address common.Address, block *big.Int) (*big.Int, error) { return big.NewInt(100000000), nil }), - backendmock.WithSuggestGasPriceFunc(func(ctx context.Context) (*big.Int, error) { - return big.NewInt(1), nil + backendmock.WithSuggestedFeeAndTipFunc(func(ctx context.Context, gasPrice *big.Int, boostPercent int) (*big.Int, *big.Int, error) { + return big.NewInt(1), big.NewInt(2), nil }), }, }) diff --git a/pkg/node/chain.go b/pkg/node/chain.go index 5931c8dde8e..c1f00dd61bc 100644 --- a/pkg/node/chain.go +++ b/pkg/node/chain.go @@ -32,7 +32,6 @@ import ( "github.com/ethersphere/bee/v2/pkg/storage" "github.com/ethersphere/bee/v2/pkg/transaction" "github.com/ethersphere/bee/v2/pkg/transaction/wrapped" - "github.com/ethersphere/go-sw3-abi/sw3abi" "github.com/prometheus/client_golang/prometheus" ) @@ -53,6 +52,7 @@ func InitChain( signer crypto.Signer, pollingInterval time.Duration, chainEnabled bool, + minimumGasTipCap uint64, ) (transaction.Backend, common.Address, int64, transaction.Monitor, transaction.Service, error) { var backend transaction.Backend = &noOpChainBackend{ chainID: oChainID, @@ -74,7 +74,7 @@ func InitChain( logger.Info("connected to blockchain backend", "version", versionString) - backend = wrapped.NewBackend(ethclient.NewClient(rpcClient)) + backend = wrapped.NewBackend(ethclient.NewClient(rpcClient), minimumGasTipCap) } chainID, err := backend.ChainID(ctx) @@ -346,19 +346,10 @@ type noOpChainBackend struct { chainID int64 } -// BlockByNumber implements transaction.Backend. -func (m *noOpChainBackend) BlockByNumber(ctx context.Context, number *big.Int) (*types.Block, error) { - return nil, postagecontract.ErrChainDisabled -} - func (m noOpChainBackend) Metrics() []prometheus.Collector { return nil } -func (m noOpChainBackend) CodeAt(context.Context, common.Address, *big.Int) ([]byte, error) { - return common.FromHex(sw3abi.SimpleSwapFactoryDeployedBinv0_6_9), nil -} - func (m noOpChainBackend) CallContract(context.Context, ethereum.CallMsg, *big.Int) ([]byte, error) { return nil, errors.New("disabled chain backend") } @@ -373,12 +364,12 @@ func (m noOpChainBackend) PendingNonceAt(context.Context, common.Address) (uint6 panic("chain no op: PendingNonceAt") } -func (m noOpChainBackend) SuggestGasPrice(context.Context) (*big.Int, error) { - panic("chain no op: SuggestGasPrice") +func (m noOpChainBackend) SuggestedFeeAndTip(ctx context.Context, gasPrice *big.Int, boostPercent int) (*big.Int, *big.Int, error) { + panic("chain no op: SuggestedFeeAndTip") } func (m noOpChainBackend) SuggestGasTipCap(context.Context) (*big.Int, error) { - panic("chain no op: SuggestGasPrice") + panic("chain no op: SuggestGasTipCap") } func (m noOpChainBackend) EstimateGas(context.Context, ethereum.CallMsg) (uint64, error) { @@ -419,6 +410,4 @@ func (m noOpChainBackend) ChainID(context.Context) (*big.Int, error) { return big.NewInt(m.chainID), nil } -func (m noOpChainBackend) Close() error { - return nil -} +func (m noOpChainBackend) Close() {} diff --git a/pkg/node/node.go b/pkg/node/node.go index c828c342c98..477fe7c230d 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -106,7 +106,6 @@ type Bee struct { pullSyncCloser io.Closer pssCloser io.Closer gsocCloser io.Closer - ethClientCloser io.Closer transactionMonitorCloser io.Closer transactionCloser io.Closer listenerCloser io.Closer @@ -121,6 +120,7 @@ type Bee struct { shutdownMutex sync.Mutex syncingStopped *syncutil.Signaler accesscontrolCloser io.Closer + ethClientCloser func() } type Options struct { @@ -145,6 +145,7 @@ type Options struct { EnableWS bool FullNodeMode bool Logger log.Logger + MinimumGasTipCap uint64 MinimumStorageRadius uint MutexProfile bool NATAddr string @@ -390,11 +391,13 @@ func NewBee( o.ChainID, signer, o.BlockTime, - chainEnabled) + chainEnabled, + o.MinimumGasTipCap, + ) if err != nil { return nil, fmt.Errorf("init chain: %w", err) } - b.ethClientCloser = chainBackend + b.ethClientCloser = chainBackend.Close logger.Info("using chain with network network", "chain_id", chainID, "network_id", networkID) @@ -1384,7 +1387,10 @@ func (b *Bee) Shutdown() error { wg.Wait() - tryClose(b.ethClientCloser, "eth client") + if b.ethClientCloser != nil { + b.ethClientCloser() + } + tryClose(b.accesscontrolCloser, "accesscontrol") tryClose(b.tracerCloser, "tracer") tryClose(b.topologyCloser, "topology driver") diff --git a/pkg/settlement/swap/chequebook/init.go b/pkg/settlement/swap/chequebook/init.go index a89553e0f1c..8903a17c8bf 100644 --- a/pkg/settlement/swap/chequebook/init.go +++ b/pkg/settlement/swap/chequebook/init.go @@ -60,12 +60,12 @@ func checkBalance( minimumEth := big.NewInt(0) if gasPrice == nil { - gasPrice, err = swapBackend.SuggestGasPrice(timeoutCtx) + gasPrice, _, err = swapBackend.SuggestedFeeAndTip(timeoutCtx, gasPrice, 0) if err != nil { return err } - minimumEth = gasPrice.Mul(gasPrice, big.NewInt(250000)) + minimumEth = new(big.Int).Mul(gasPrice, big.NewInt(250000)) } insufficientERC20 := erc20Balance.Cmp(swapInitialDeposit) < 0 diff --git a/pkg/storageincentives/agent.go b/pkg/storageincentives/agent.go index a87540d390d..04c906c132a 100644 --- a/pkg/storageincentives/agent.go +++ b/pkg/storageincentives/agent.go @@ -46,7 +46,7 @@ type ChainBackend interface { BlockNumber(context.Context) (uint64, error) HeaderByNumber(context.Context, *big.Int) (*types.Header, error) BalanceAt(ctx context.Context, address common.Address, block *big.Int) (*big.Int, error) - SuggestGasPrice(ctx context.Context) (*big.Int, error) + SuggestedFeeAndTip(ctx context.Context, gasPrice *big.Int, boostPercent int) (*big.Int, *big.Int, error) } type Health interface { @@ -604,7 +604,7 @@ func (a *Agent) HasEnoughFundsToPlay(ctx context.Context) (*big.Int, bool, error return nil, false, err } - price, err := a.backend.SuggestGasPrice(ctx) + price, _, err := a.backend.SuggestedFeeAndTip(ctx, nil, redistribution.BoostTipPercent) if err != nil { return nil, false, err } diff --git a/pkg/storageincentives/agent_test.go b/pkg/storageincentives/agent_test.go index 8af6b1d463e..88db0e3b3d4 100644 --- a/pkg/storageincentives/agent_test.go +++ b/pkg/storageincentives/agent_test.go @@ -242,8 +242,8 @@ func (m *mockchainBackend) BalanceAt(ctx context.Context, address common.Address return m.balance, nil } -func (m *mockchainBackend) SuggestGasPrice(ctx context.Context) (*big.Int, error) { - return big.NewInt(4), nil +func (m *mockchainBackend) SuggestedFeeAndTip(ctx context.Context, gasPrice *big.Int, boostPercent int) (*big.Int, *big.Int, error) { + return big.NewInt(4), big.NewInt(5), nil } type contractCall int diff --git a/pkg/storageincentives/redistribution/redistribution.go b/pkg/storageincentives/redistribution/redistribution.go index 4657c81cbff..52b2245db75 100644 --- a/pkg/storageincentives/redistribution/redistribution.go +++ b/pkg/storageincentives/redistribution/redistribution.go @@ -17,7 +17,10 @@ import ( "github.com/ethersphere/bee/v2/pkg/transaction" ) -const loggerName = "redistributionContract" +const ( + loggerName = "redistributionContract" + BoostTipPercent = 50 +) type Contract interface { ReserveSalt(context.Context) ([]byte, error) @@ -117,7 +120,7 @@ func (c *contract) Claim(ctx context.Context, proofs ChunkInclusionProofs) (comm Value: big.NewInt(0), Description: "claim win transaction", } - txHash, err := c.sendAndWait(ctx, request, transaction.RedistributionTipBoostPercent) + txHash, err := c.sendAndWait(ctx, request, BoostTipPercent) if err != nil { return txHash, fmt.Errorf("claim: %w", err) } @@ -140,7 +143,7 @@ func (c *contract) Commit(ctx context.Context, obfusHash []byte, round uint64) ( Value: big.NewInt(0), Description: "commit transaction", } - txHash, err := c.sendAndWait(ctx, request, transaction.RedistributionTipBoostPercent) + txHash, err := c.sendAndWait(ctx, request, BoostTipPercent) if err != nil { return txHash, fmt.Errorf("commit: obfusHash %v: %w", common.BytesToHash(obfusHash), err) } @@ -163,7 +166,7 @@ func (c *contract) Reveal(ctx context.Context, storageDepth uint8, reserveCommit Value: big.NewInt(0), Description: "reveal transaction", } - txHash, err := c.sendAndWait(ctx, request, transaction.RedistributionTipBoostPercent) + txHash, err := c.sendAndWait(ctx, request, BoostTipPercent) if err != nil { return txHash, fmt.Errorf("reveal: storageDepth %d reserveCommitmentHash %v RandomNonce %v: %w", storageDepth, common.BytesToHash(reserveCommitmentHash), common.BytesToHash(RandomNonce), err) } diff --git a/pkg/transaction/backend.go b/pkg/transaction/backend.go index 0bf38c35975..075dbfe19b8 100644 --- a/pkg/transaction/backend.go +++ b/pkg/transaction/backend.go @@ -14,27 +14,13 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethersphere/bee/v2/pkg/log" + "github.com/ethersphere/bee/v2/pkg/transaction/backend" ) // Backend is the minimum of blockchain backend functions we need. type Backend interface { - CodeAt(ctx context.Context, contract common.Address, blockNumber *big.Int) ([]byte, error) - CallContract(ctx context.Context, call ethereum.CallMsg, blockNumber *big.Int) ([]byte, error) - HeaderByNumber(ctx context.Context, number *big.Int) (*types.Header, error) - PendingNonceAt(ctx context.Context, account common.Address) (uint64, error) - SuggestGasPrice(ctx context.Context) (*big.Int, error) - SuggestGasTipCap(ctx context.Context) (*big.Int, error) - EstimateGas(ctx context.Context, call ethereum.CallMsg) (gas uint64, err error) - SendTransaction(ctx context.Context, tx *types.Transaction) error - TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error) - TransactionByHash(ctx context.Context, hash common.Hash) (tx *types.Transaction, isPending bool, err error) - BlockNumber(ctx context.Context) (uint64, error) - BlockByNumber(ctx context.Context, number *big.Int) (*types.Block, error) - BalanceAt(ctx context.Context, address common.Address, block *big.Int) (*big.Int, error) - NonceAt(ctx context.Context, account common.Address, blockNumber *big.Int) (uint64, error) - FilterLogs(ctx context.Context, query ethereum.FilterQuery) ([]types.Log, error) - ChainID(ctx context.Context) (*big.Int, error) - Close() error + backend.Geth + SuggestedFeeAndTip(ctx context.Context, gasPrice *big.Int, boostPercent int) (*big.Int, *big.Int, error) } // IsSynced will check if we are synced with the given blockchain backend. This @@ -42,11 +28,7 @@ type Backend interface { // with the given maxDelay as the maximum duration we can be behind the block // time. func IsSynced(ctx context.Context, backend Backend, maxDelay time.Duration) (bool, time.Time, error) { - number, err := backend.BlockNumber(ctx) - if err != nil { - return false, time.Time{}, err - } - header, err := backend.HeaderByNumber(ctx, big.NewInt(int64(number))) + header, err := backend.HeaderByNumber(ctx, nil) if errors.Is(err, ethereum.NotFound) { return false, time.Time{}, nil } diff --git a/pkg/transaction/backend/backend.go b/pkg/transaction/backend/backend.go new file mode 100644 index 00000000000..67323d14e45 --- /dev/null +++ b/pkg/transaction/backend/backend.go @@ -0,0 +1,32 @@ +// Copyright 2025 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package backend + +import ( + "context" + "math/big" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" +) + +// Geth is the interface that an ethclient.Client satisfies. +type Geth interface { + BalanceAt(ctx context.Context, account common.Address, blockNumber *big.Int) (*big.Int, error) + BlockNumber(ctx context.Context) (uint64, error) + CallContract(ctx context.Context, msg ethereum.CallMsg, blockNumber *big.Int) ([]byte, error) + ChainID(ctx context.Context) (*big.Int, error) + Close() + EstimateGas(ctx context.Context, msg ethereum.CallMsg) (uint64, error) + FilterLogs(ctx context.Context, q ethereum.FilterQuery) ([]types.Log, error) + HeaderByNumber(ctx context.Context, number *big.Int) (*types.Header, error) + NonceAt(ctx context.Context, account common.Address, blockNumber *big.Int) (uint64, error) + PendingNonceAt(ctx context.Context, account common.Address) (uint64, error) + SendTransaction(ctx context.Context, tx *types.Transaction) error + SuggestGasTipCap(ctx context.Context) (*big.Int, error) + TransactionByHash(ctx context.Context, hash common.Hash) (tx *types.Transaction, isPending bool, err error) + TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error) +} diff --git a/pkg/transaction/backend_test.go b/pkg/transaction/backend_test.go index d9d09e55b3c..313db5d9b9f 100644 --- a/pkg/transaction/backend_test.go +++ b/pkg/transaction/backend_test.go @@ -22,7 +22,6 @@ func TestIsSynced(t *testing.T) { maxDelay := 10 * time.Second now := time.Now().UTC() ctx := context.Background() - blockNumber := uint64(100) t.Run("synced", func(t *testing.T) { t.Parallel() @@ -30,12 +29,9 @@ func TestIsSynced(t *testing.T) { synced, _, err := transaction.IsSynced( ctx, backendmock.New( - backendmock.WithBlockNumberFunc(func(c context.Context) (uint64, error) { - return blockNumber, nil - }), backendmock.WithHeaderbyNumberFunc(func(ctx context.Context, number *big.Int) (*types.Header, error) { - if number.Uint64() != blockNumber { - return nil, errors.New("called with wrong block number") + if number != nil { + return nil, errors.New("latest block should be called with nil") } return &types.Header{ Time: uint64(now.Unix()), @@ -58,12 +54,9 @@ func TestIsSynced(t *testing.T) { synced, _, err := transaction.IsSynced( ctx, backendmock.New( - backendmock.WithBlockNumberFunc(func(c context.Context) (uint64, error) { - return blockNumber, nil - }), backendmock.WithHeaderbyNumberFunc(func(ctx context.Context, number *big.Int) (*types.Header, error) { - if number.Uint64() != blockNumber { - return nil, errors.New("called with wrong block number") + if number != nil { + return nil, errors.New("latest block should be called with nil") } return &types.Header{ Time: uint64(now.Add(-maxDelay).Unix()), @@ -87,12 +80,9 @@ func TestIsSynced(t *testing.T) { _, _, err := transaction.IsSynced( ctx, backendmock.New( - backendmock.WithBlockNumberFunc(func(c context.Context) (uint64, error) { - return blockNumber, nil - }), backendmock.WithHeaderbyNumberFunc(func(ctx context.Context, number *big.Int) (*types.Header, error) { - if number.Uint64() != blockNumber { - return nil, errors.New("called with wrong block number") + if number != nil { + return nil, errors.New("latest block should be called with nil") } return nil, expectedErr }), diff --git a/pkg/transaction/backendmock/backend.go b/pkg/transaction/backendmock/backend.go index fea46f3936b..25166a898e0 100644 --- a/pkg/transaction/backendmock/backend.go +++ b/pkg/transaction/backendmock/backend.go @@ -16,29 +16,20 @@ import ( ) type backendMock struct { - codeAt func(ctx context.Context, contract common.Address, blockNumber *big.Int) ([]byte, error) callContract func(ctx context.Context, call ethereum.CallMsg, blockNumber *big.Int) ([]byte, error) sendTransaction func(ctx context.Context, tx *types.Transaction) error - suggestGasPrice func(ctx context.Context) (*big.Int, error) + suggestedFeeAndTip func(ctx context.Context, gasPrice *big.Int, boostPercent int) (*big.Int, *big.Int, error) suggestGasTipCap func(ctx context.Context) (*big.Int, error) estimateGas func(ctx context.Context, call ethereum.CallMsg) (gas uint64, err error) transactionReceipt func(ctx context.Context, txHash common.Hash) (*types.Receipt, error) pendingNonceAt func(ctx context.Context, account common.Address) (uint64, error) transactionByHash func(ctx context.Context, hash common.Hash) (tx *types.Transaction, isPending bool, err error) blockNumber func(ctx context.Context) (uint64, error) - blockByNumber func(ctx context.Context, number *big.Int) (*types.Block, error) headerByNumber func(ctx context.Context, number *big.Int) (*types.Header, error) balanceAt func(ctx context.Context, address common.Address, block *big.Int) (*big.Int, error) nonceAt func(ctx context.Context, account common.Address, blockNumber *big.Int) (uint64, error) } -func (m *backendMock) CodeAt(ctx context.Context, contract common.Address, blockNumber *big.Int) ([]byte, error) { - if m.codeAt != nil { - return m.codeAt(ctx, contract, blockNumber) - } - return nil, errors.New("not implemented") -} - func (m *backendMock) CallContract(ctx context.Context, call ethereum.CallMsg, blockNumber *big.Int) ([]byte, error) { if m.callContract != nil { return m.callContract(ctx, call, blockNumber) @@ -46,10 +37,6 @@ func (m *backendMock) CallContract(ctx context.Context, call ethereum.CallMsg, b return nil, errors.New("not implemented") } -func (*backendMock) PendingCodeAt(ctx context.Context, account common.Address) ([]byte, error) { - return nil, errors.New("not implemented") -} - func (m *backendMock) PendingNonceAt(ctx context.Context, account common.Address) (uint64, error) { if m.pendingNonceAt != nil { return m.pendingNonceAt(ctx, account) @@ -57,11 +44,11 @@ func (m *backendMock) PendingNonceAt(ctx context.Context, account common.Address return 0, errors.New("not implemented") } -func (m *backendMock) SuggestGasPrice(ctx context.Context) (*big.Int, error) { - if m.suggestGasPrice != nil { - return m.suggestGasPrice(ctx) +func (m *backendMock) SuggestedFeeAndTip(ctx context.Context, gasPrice *big.Int, boostPercent int) (*big.Int, *big.Int, error) { + if m.suggestedFeeAndTip != nil { + return m.suggestedFeeAndTip(ctx, gasPrice, boostPercent) } - return nil, errors.New("not implemented") + return nil, nil, errors.New("not implemented") } func (m *backendMock) EstimateGas(ctx context.Context, call ethereum.CallMsg) (gas uint64, err error) { @@ -82,10 +69,6 @@ func (*backendMock) FilterLogs(ctx context.Context, query ethereum.FilterQuery) return nil, errors.New("not implemented") } -func (*backendMock) SubscribeFilterLogs(ctx context.Context, query ethereum.FilterQuery, ch chan<- types.Log) (ethereum.Subscription, error) { - return nil, errors.New("not implemented") -} - func (m *backendMock) TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error) { if m.transactionReceipt != nil { return m.transactionReceipt(ctx, txHash) @@ -107,13 +90,6 @@ func (m *backendMock) BlockNumber(ctx context.Context) (uint64, error) { return 0, errors.New("not implemented") } -func (m *backendMock) BlockByNumber(ctx context.Context, number *big.Int) (*types.Block, error) { - if m.blockByNumber != nil { - return m.blockByNumber(ctx, number) - } - return nil, errors.New("not implemented") -} - func (m *backendMock) HeaderByNumber(ctx context.Context, number *big.Int) (*types.Header, error) { if m.headerByNumber != nil { return m.headerByNumber(ctx, number) @@ -146,9 +122,7 @@ func (m *backendMock) ChainID(ctx context.Context) (*big.Int, error) { return nil, errors.New("not implemented") } -func (m *backendMock) Close() error { - return nil -} +func (m *backendMock) Close() {} func New(opts ...Option) transaction.Backend { mock := new(backendMock) @@ -173,12 +147,6 @@ func WithCallContractFunc(f func(ctx context.Context, call ethereum.CallMsg, blo }) } -func WithCodeAtFunc(f func(ctx context.Context, contract common.Address, blockNumber *big.Int) ([]byte, error)) Option { - return optionFunc(func(s *backendMock) { - s.codeAt = f - }) -} - func WithBalanceAt(f func(ctx context.Context, address common.Address, block *big.Int) (*big.Int, error)) Option { return optionFunc(func(s *backendMock) { s.balanceAt = f @@ -191,9 +159,9 @@ func WithPendingNonceAtFunc(f func(ctx context.Context, account common.Address) }) } -func WithSuggestGasPriceFunc(f func(ctx context.Context) (*big.Int, error)) Option { +func WithSuggestedFeeAndTipFunc(f func(ctx context.Context, gasPrice *big.Int, boostPercent int) (*big.Int, *big.Int, error)) Option { return optionFunc(func(s *backendMock) { - s.suggestGasPrice = f + s.suggestedFeeAndTip = f }) } @@ -221,12 +189,6 @@ func WithTransactionByHashFunc(f func(ctx context.Context, txHash common.Hash) ( }) } -func WithBlockByNumberFunc(f func(ctx context.Context, number *big.Int) (*types.Block, error)) Option { - return optionFunc(func(s *backendMock) { - s.blockByNumber = f - }) -} - func WithSendTransactionFunc(f func(ctx context.Context, tx *types.Transaction) error) Option { return optionFunc(func(s *backendMock) { s.sendTransaction = f diff --git a/pkg/transaction/backendsimulation/backend.go b/pkg/transaction/backendsimulation/backend.go index f552b47b9bf..47480fd3994 100644 --- a/pkg/transaction/backendsimulation/backend.go +++ b/pkg/transaction/backendsimulation/backend.go @@ -83,28 +83,16 @@ func (m *simulatedBackend) advanceBlock() { } } -func (m *simulatedBackend) BlockByNumber(ctx context.Context, number *big.Int) (*types.Block, error) { - return nil, errors.New("not implemented") -} - -func (m *simulatedBackend) CodeAt(ctx context.Context, contract common.Address, blockNumber *big.Int) ([]byte, error) { - return nil, errors.New("not implemented") -} - func (*simulatedBackend) CallContract(ctx context.Context, call ethereum.CallMsg, blockNumber *big.Int) ([]byte, error) { return nil, errors.New("not implemented") } -func (*simulatedBackend) PendingCodeAt(ctx context.Context, account common.Address) ([]byte, error) { - return nil, errors.New("not implemented") -} - func (m *simulatedBackend) PendingNonceAt(ctx context.Context, account common.Address) (uint64, error) { return 0, errors.New("not implemented") } -func (m *simulatedBackend) SuggestGasPrice(ctx context.Context) (*big.Int, error) { - return nil, errors.New("not implemented") +func (m *simulatedBackend) SuggestedFeeAndTip(ctx context.Context, gasPrice *big.Int, boostPercent int) (*big.Int, *big.Int, error) { + return nil, nil, errors.New("not implemented") } func (m *simulatedBackend) EstimateGas(ctx context.Context, call ethereum.CallMsg) (gas uint64, err error) { @@ -119,10 +107,6 @@ func (*simulatedBackend) FilterLogs(ctx context.Context, query ethereum.FilterQu return nil, errors.New("not implemented") } -func (*simulatedBackend) SubscribeFilterLogs(ctx context.Context, query ethereum.FilterQuery, ch chan<- types.Log) (ethereum.Subscription, error) { - return nil, errors.New("not implemented") -} - func (m *simulatedBackend) TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error) { receipt, ok := m.receipts[txHash] if ok { @@ -166,6 +150,4 @@ func (m *simulatedBackend) ChainID(ctx context.Context) (*big.Int, error) { return nil, errors.New("not implemented") } -func (m *simulatedBackend) Close() error { - return nil -} +func (m *simulatedBackend) Close() {} diff --git a/pkg/transaction/transaction.go b/pkg/transaction/transaction.go index 2d911e27261..da71a2150e3 100644 --- a/pkg/transaction/transaction.go +++ b/pkg/transaction/transaction.go @@ -41,14 +41,11 @@ var ( ErrTransactionReverted = errors.New("transaction reverted") ErrUnknownTransaction = errors.New("unknown transaction") ErrAlreadyImported = errors.New("already imported") - ErrEIP1559NotSupported = errors.New("network does not appear to support EIP-1559 (no baseFee)") ) const ( - DefaultGasLimit = 1_000_000 - DefaultTipBoostPercent = 25 - MinimumGasTipCap = 1_500_000_000 // 1.5 Gwei - RedistributionTipBoostPercent = 50 + DefaultGasLimit = 1_000_000 + DefaultTipBoostPercent = 25 ) // TxRequest describes a request for a transaction that can be executed. @@ -309,7 +306,7 @@ func (t *transactionService) prepareTransaction(ctx context.Context, request *Tx notice that gas price does not exceed 20 as defined by max fee. */ - gasFeeCap, gasTipCap, err := t.suggestedFeeAndTip(ctx, request.GasPrice, boostPercent) + gasFeeCap, gasTipCap, err := t.backend.SuggestedFeeAndTip(ctx, request.GasPrice, boostPercent) if err != nil { return nil, err } @@ -326,51 +323,6 @@ func (t *transactionService) prepareTransaction(ctx context.Context, request *Tx }), nil } -func (t *transactionService) suggestedFeeAndTip(ctx context.Context, gasPrice *big.Int, boostPercent int) (*big.Int, *big.Int, error) { - gasTipCap, err := t.backend.SuggestGasTipCap(ctx) - if err != nil { - return nil, nil, err - } - - multiplier := big.NewInt(int64(boostPercent) + 100) - gasTipCap = new(big.Int).Div(new(big.Int).Mul(gasTipCap, multiplier), big.NewInt(100)) - - minimumTip := big.NewInt(MinimumGasTipCap) - if gasTipCap.Cmp(minimumTip) < 0 { - gasTipCap = new(big.Int).Set(minimumTip) - } - - var gasFeeCap *big.Int - - if gasPrice == nil { - latestBlockHeader, err := t.backend.HeaderByNumber(ctx, nil) - if err != nil { - return nil, nil, fmt.Errorf("failed to get latest block: %w", err) - } - - if latestBlockHeader.BaseFee == nil { - return nil, nil, ErrEIP1559NotSupported - } - - // gasFeeCap = (2 * baseFee) + gasTipCap - gasFeeCap = new(big.Int).Add( - new(big.Int).Mul(latestBlockHeader.BaseFee, big.NewInt(2)), - gasTipCap, - ) - } else { - gasFeeCap = new(big.Int).Set(gasPrice) - } - - if gasTipCap.Cmp(gasFeeCap) > 0 { - t.logger.Warning("gas tip cap is higher than gas fee cap, using gas fee cap as gas tip cap", "gas_tip_cap", gasTipCap, "gas_fee_cap", gasFeeCap) - gasTipCap = new(big.Int).Set(gasFeeCap) - } - - t.logger.Debug("prepare transaction", "gas_max_fee", gasFeeCap, "gas_max_tip", gasTipCap) - - return gasFeeCap, gasTipCap, nil -} - func storedTransactionKey(txHash common.Hash) string { return fmt.Sprintf("%s%x", storedTransactionPrefix, txHash) } @@ -394,7 +346,7 @@ func (t *transactionService) nextNonce(ctx context.Context) (uint64, error) { // PendingNonceAt returns the nonce we should use, but we will // compare this to our pending tx list, therefore the -1. - var maxNonce = onchainNonce - 1 + maxNonce := onchainNonce - 1 for _, txHash := range pendingTxs { trx, _, err := t.backend.TransactionByHash(ctx, txHash) if err != nil { @@ -441,7 +393,7 @@ func (t *transactionService) WatchSentTransaction(txHash common.Hash) (<-chan ty } func (t *transactionService) PendingTransactions() ([]common.Hash, error) { - var txHashes = make([]common.Hash, 0) + txHashes := make([]common.Hash, 0) err := t.store.Iterate(pendingTransactionPrefix, func(key, value []byte) (stop bool, err error) { txHash := common.HexToHash(strings.TrimPrefix(string(key), pendingTransactionPrefix)) txHashes = append(txHashes, txHash) @@ -491,7 +443,7 @@ func (t *transactionService) ResendTransaction(ctx context.Context, txHash commo return err } - gasFeeCap, gasTipCap, err := t.suggestedFeeAndTip(ctx, sctx.GetGasPrice(ctx), storedTransaction.GasTipBoost) + gasFeeCap, gasTipCap, err := t.backend.SuggestedFeeAndTip(ctx, sctx.GetGasPrice(ctx), storedTransaction.GasTipBoost) if err != nil { return err } @@ -531,7 +483,7 @@ func (t *transactionService) CancelTransaction(ctx context.Context, originalTxHa return common.Hash{}, err } - gasFeeCap, gasTipCap, err := t.suggestedFeeAndTip(ctx, sctx.GetGasPrice(ctx), 0) + gasFeeCap, gasTipCap, err := t.backend.SuggestedFeeAndTip(ctx, sctx.GetGasPrice(ctx), 0) if err != nil { return common.Hash{}, err } diff --git a/pkg/transaction/transaction_test.go b/pkg/transaction/transaction_test.go index d8b830e5780..079e341dd27 100644 --- a/pkg/transaction/transaction_test.go +++ b/pkg/transaction/transaction_test.go @@ -31,7 +31,7 @@ import ( ) var ( - minimumTip = big.NewInt(transaction.MinimumGasTipCap) + minimumTip = big.NewInt(1_500_000_000) baseFee = big.NewInt(3_000_000_000) ) @@ -75,6 +75,39 @@ func signerMockForTransaction(t *testing.T, signedTx *types.Transaction, sender ) } +func checkStoredTransaction(t *testing.T, transactionService transaction.Service, txHash common.Hash, request *transaction.TxRequest, recipient common.Address, gasLimit uint64, gasPrice *big.Int, nonce uint64) { + t.Helper() + + storedTransaction, err := transactionService.StoredTransaction(txHash) + if err != nil { + t.Fatal(err) + } + + if storedTransaction.To == nil || *storedTransaction.To != recipient { + t.Fatalf("got wrong recipient in stored transaction. wanted %x, got %x", recipient, storedTransaction.To) + } + + if !bytes.Equal(storedTransaction.Data, request.Data) { + t.Fatalf("got wrong data in stored transaction. wanted %x, got %x", request.Data, storedTransaction.Data) + } + + if storedTransaction.Description != request.Description { + t.Fatalf("got wrong description in stored transaction. wanted %x, got %x", request.Description, storedTransaction.Description) + } + + if storedTransaction.GasLimit != gasLimit { + t.Fatalf("got wrong gas limit in stored transaction. wanted %d, got %d", gasLimit, storedTransaction.GasLimit) + } + + if gasPrice.Cmp(storedTransaction.GasPrice) != 0 { + t.Fatalf("got wrong gas price in stored transaction. wanted %d, got %d", gasPrice, storedTransaction.GasPrice) + } + + if storedTransaction.Nonce != nonce { + t.Fatalf("got wrong nonce in stored transaction. wanted %d, got %d", nonce, storedTransaction.Nonce) + } +} + func TestTransactionSend(t *testing.T) { t.Parallel() @@ -130,11 +163,8 @@ func TestTransactionSend(t *testing.T) { backendmock.WithPendingNonceAtFunc(func(ctx context.Context, account common.Address) (uint64, error) { return nonce - 1, nil }), - backendmock.WithSuggestGasTipCapFunc(func(ctx context.Context) (*big.Int, error) { - return suggestedGasTip, nil - }), - backendmock.WithHeaderbyNumberFunc(func(ctx context.Context, number *big.Int) (*types.Header, error) { - return &types.Header{BaseFee: baseFee}, nil + backendmock.WithSuggestedFeeAndTipFunc(func(ctx context.Context, gasPrice *big.Int, boostPercent int) (*big.Int, *big.Int, error) { + return gasFeeCap, suggestedGasTip, nil }), ), signerMockForTransaction(t, signedTx, sender, chainID), @@ -160,34 +190,7 @@ func TestTransactionSend(t *testing.T) { t.Fatal("returning wrong transaction hash") } - storedTransaction, err := transactionService.StoredTransaction(txHash) - if err != nil { - t.Fatal(err) - } - - if storedTransaction.To == nil || *storedTransaction.To != recipient { - t.Fatalf("got wrong recipient in stored transaction. wanted %x, got %x", recipient, storedTransaction.To) - } - - if !bytes.Equal(storedTransaction.Data, request.Data) { - t.Fatalf("got wrong data in stored transaction. wanted %x, got %x", request.Data, storedTransaction.Data) - } - - if storedTransaction.Description != request.Description { - t.Fatalf("got wrong description in stored transaction. wanted %x, got %x", request.Description, storedTransaction.Description) - } - - if storedTransaction.GasLimit != gasLimit { - t.Fatalf("got wrong gas limit in stored transaction. wanted %d, got %d", gasLimit, storedTransaction.GasLimit) - } - - if gasFeeCap.Cmp(storedTransaction.GasPrice) != 0 { - t.Fatalf("got wrong gas price in stored transaction. wanted %d, got %d", gasFeeCap, storedTransaction.GasPrice) - } - - if storedTransaction.Nonce != nonce { - t.Fatalf("got wrong nonce in stored transaction. wanted %d, got %d", nonce, storedTransaction.Nonce) - } + checkStoredTransaction(t, transactionService, txHash, request, recipient, gasLimit, gasFeeCap, nonce) pending, err := transactionService.PendingTransactions() if err != nil { @@ -237,11 +240,8 @@ func TestTransactionSend(t *testing.T) { backendmock.WithPendingNonceAtFunc(func(ctx context.Context, account common.Address) (uint64, error) { return nonce - 1, nil }), - backendmock.WithSuggestGasTipCapFunc(func(ctx context.Context) (*big.Int, error) { - return suggestedGasTip, nil - }), - backendmock.WithHeaderbyNumberFunc(func(ctx context.Context, number *big.Int) (*types.Header, error) { - return &types.Header{BaseFee: baseFee}, nil + backendmock.WithSuggestedFeeAndTipFunc(func(ctx context.Context, gasPrice *big.Int, boostPercent int) (*big.Int, *big.Int, error) { + return gasFeeCap, suggestedGasTip, nil }), ), signerMockForTransaction(t, signedTx, sender, chainID), @@ -267,34 +267,7 @@ func TestTransactionSend(t *testing.T) { t.Fatal("returning wrong transaction hash") } - storedTransaction, err := transactionService.StoredTransaction(txHash) - if err != nil { - t.Fatal(err) - } - - if storedTransaction.To == nil || *storedTransaction.To != recipient { - t.Fatalf("got wrong recipient in stored transaction. wanted %x, got %x", recipient, storedTransaction.To) - } - - if !bytes.Equal(storedTransaction.Data, request.Data) { - t.Fatalf("got wrong data in stored transaction. wanted %x, got %x", request.Data, storedTransaction.Data) - } - - if storedTransaction.Description != request.Description { - t.Fatalf("got wrong description in stored transaction. wanted %x, got %x", request.Description, storedTransaction.Description) - } - - if storedTransaction.GasLimit != gasLimit { - t.Fatalf("got wrong gas limit in stored transaction. wanted %d, got %d", gasLimit, storedTransaction.GasLimit) - } - - if gasFeeCap.Cmp(storedTransaction.GasPrice) != 0 { - t.Fatalf("got wrong gas price in stored transaction. wanted %d, got %d", gasFeeCap, storedTransaction.GasPrice) - } - - if storedTransaction.Nonce != nonce { - t.Fatalf("got wrong nonce in stored transaction. wanted %d, got %d", nonce, storedTransaction.Nonce) - } + checkStoredTransaction(t, transactionService, txHash, request, recipient, gasLimit, gasFeeCap, nonce) pending, err := transactionService.PendingTransactions() if err != nil { @@ -353,11 +326,8 @@ func TestTransactionSend(t *testing.T) { backendmock.WithPendingNonceAtFunc(func(ctx context.Context, account common.Address) (uint64, error) { return nonce - 1, nil }), - backendmock.WithSuggestGasTipCapFunc(func(ctx context.Context) (*big.Int, error) { - return suggestedGasTip, nil - }), - backendmock.WithHeaderbyNumberFunc(func(ctx context.Context, number *big.Int) (*types.Header, error) { - return &types.Header{BaseFee: baseFee}, nil + backendmock.WithSuggestedFeeAndTipFunc(func(ctx context.Context, gasPrice *big.Int, boostPercent int) (*big.Int, *big.Int, error) { + return gasFeeCapWithBoost, suggestedGasTip, nil }), ), signerMockForTransaction(t, signedTx, sender, chainID), @@ -383,34 +353,7 @@ func TestTransactionSend(t *testing.T) { t.Fatal("returning wrong transaction hash") } - storedTransaction, err := transactionService.StoredTransaction(txHash) - if err != nil { - t.Fatal(err) - } - - if storedTransaction.To == nil || *storedTransaction.To != recipient { - t.Fatalf("got wrong recipient in stored transaction. wanted %x, got %x", recipient, storedTransaction.To) - } - - if !bytes.Equal(storedTransaction.Data, request.Data) { - t.Fatalf("got wrong data in stored transaction. wanted %x, got %x", request.Data, storedTransaction.Data) - } - - if storedTransaction.Description != request.Description { - t.Fatalf("got wrong description in stored transaction. wanted %x, got %x", request.Description, storedTransaction.Description) - } - - if storedTransaction.GasLimit != gasLimit { - t.Fatalf("got wrong gas limit in stored transaction. wanted %d, got %d", gasLimit, storedTransaction.GasLimit) - } - - if gasFeeCapWithBoost.Cmp(storedTransaction.GasPrice) != 0 { - t.Fatalf("got wrong gas price in stored transaction. wanted %d, got %d", gasFeeCapWithBoost, storedTransaction.GasPrice) - } - - if storedTransaction.Nonce != nonce { - t.Fatalf("got wrong nonce in stored transaction. wanted %d, got %d", nonce, storedTransaction.Nonce) - } + checkStoredTransaction(t, transactionService, txHash, request, recipient, gasLimit, gasFeeCapWithBoost, nonce) pending, err := transactionService.PendingTransactions() if err != nil { @@ -465,11 +408,8 @@ func TestTransactionSend(t *testing.T) { backendmock.WithPendingNonceAtFunc(func(ctx context.Context, account common.Address) (uint64, error) { return nonce, nil }), - backendmock.WithSuggestGasTipCapFunc(func(ctx context.Context) (*big.Int, error) { - return suggestedGasTip, nil - }), - backendmock.WithHeaderbyNumberFunc(func(ctx context.Context, number *big.Int) (*types.Header, error) { - return &types.Header{BaseFee: baseFee}, nil + backendmock.WithSuggestedFeeAndTipFunc(func(ctx context.Context, gasPrice *big.Int, boostPercent int) (*big.Int, *big.Int, error) { + return gasFeeCap, suggestedGasTip, nil }), ), signerMockForTransaction(t, signedTx, sender, chainID), @@ -533,11 +473,8 @@ func TestTransactionSend(t *testing.T) { backendmock.WithPendingNonceAtFunc(func(ctx context.Context, account common.Address) (uint64, error) { return nextNonce, nil }), - backendmock.WithSuggestGasTipCapFunc(func(ctx context.Context) (*big.Int, error) { - return suggestedGasTip, nil - }), - backendmock.WithHeaderbyNumberFunc(func(ctx context.Context, number *big.Int) (*types.Header, error) { - return &types.Header{BaseFee: baseFee}, nil + backendmock.WithSuggestedFeeAndTipFunc(func(ctx context.Context, gasPrice *big.Int, boostPercent int) (*big.Int, *big.Int, error) { + return gasFeeCap, suggestedGasTip, nil }), ), signerMockForTransaction(t, signedTx, sender, chainID), @@ -602,11 +539,8 @@ func TestTransactionSend(t *testing.T) { backendmock.WithPendingNonceAtFunc(func(ctx context.Context, account common.Address) (uint64, error) { return nextNonce, nil }), - backendmock.WithSuggestGasTipCapFunc(func(ctx context.Context) (*big.Int, error) { - return suggestedGasTip, nil - }), - backendmock.WithHeaderbyNumberFunc(func(ctx context.Context, number *big.Int) (*types.Header, error) { - return &types.Header{BaseFee: baseFee}, nil + backendmock.WithSuggestedFeeAndTipFunc(func(ctx context.Context, gasPrice *big.Int, boostPercent int) (*big.Int, *big.Int, error) { + return customGasFeeCap, customGasFeeCap, nil }), ), signerMockForTransaction(t, signedTx, sender, chainID), @@ -748,11 +682,8 @@ func TestTransactionResend(t *testing.T) { } return nil }), - backendmock.WithSuggestGasTipCapFunc(func(ctx context.Context) (*big.Int, error) { - return gasTip, nil - }), - backendmock.WithHeaderbyNumberFunc(func(ctx context.Context, number *big.Int) (*types.Header, error) { - return &types.Header{BaseFee: baseFee}, nil + backendmock.WithSuggestedFeeAndTipFunc(func(ctx context.Context, gasPrice *big.Int, boostPercent int) (*big.Int, *big.Int, error) { + return gasFeeCap, gasTip, nil }), ), signerMockForTransaction(t, signedTx, recipient, chainID), @@ -838,11 +769,8 @@ func TestTransactionCancel(t *testing.T) { } return nil }), - backendmock.WithSuggestGasTipCapFunc(func(ctx context.Context) (*big.Int, error) { - return gasTip, nil - }), - backendmock.WithHeaderbyNumberFunc(func(ctx context.Context, number *big.Int) (*types.Header, error) { - return &types.Header{BaseFee: baseFee}, nil + backendmock.WithSuggestedFeeAndTipFunc(func(ctx context.Context, gasPrice *big.Int, boostPercent int) (*big.Int, *big.Int, error) { + return fee, minimumTip, nil }), ), signerMockForTransaction(t, cancelTx, recipient, chainID), @@ -879,7 +807,7 @@ func TestTransactionCancel(t *testing.T) { Value: big.NewInt(0), Gas: 21000, GasFeeCap: gasFeeCap, - GasTipCap: gasTip, + GasTipCap: gasTipCap, Data: []byte{}, }) @@ -891,8 +819,8 @@ func TestTransactionCancel(t *testing.T) { } return nil }), - backendmock.WithSuggestGasTipCapFunc(func(ctx context.Context) (*big.Int, error) { - return gasTip, nil + backendmock.WithSuggestedFeeAndTipFunc(func(ctx context.Context, gasPrice *big.Int, boostPercent int) (*big.Int, *big.Int, error) { + return gasFee, gasTip, nil }), ), signerMockForTransaction(t, cancelTx, recipient, chainID), diff --git a/pkg/transaction/wrapped/fee.go b/pkg/transaction/wrapped/fee.go new file mode 100644 index 00000000000..3b02b3423ec --- /dev/null +++ b/pkg/transaction/wrapped/fee.go @@ -0,0 +1,61 @@ +// Copyright 2025 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package wrapped + +import ( + "context" + "errors" + "fmt" + "math/big" +) + +const ( + percentageDivisor = 100 + baseFeeMultiplier = 2 +) + +var ( + ErrEIP1559NotSupported = errors.New("network does not appear to support EIP-1559 (no baseFee)") +) + +// SuggestedFeeAndTip calculates the recommended gasFeeCap and gasTipCap for a transaction. +func (b *wrappedBackend) SuggestedFeeAndTip(ctx context.Context, gasPrice *big.Int, boostPercent int) (*big.Int, *big.Int, error) { + if gasPrice != nil { + // 1. gasFeeCap: The absolute maximum price per gas does not exceed the user's specified price. + // 2. gasTipCap: The entire amount (gasPrice - baseFee) can be used as a priority fee. + return new(big.Int).Set(gasPrice), new(big.Int).Set(gasPrice), nil + } + + gasTipCap, err := b.backend.SuggestGasTipCap(ctx) + if err != nil { + return nil, nil, fmt.Errorf("failed to suggest gas tip cap: %w", err) + } + + if boostPercent != 0 { + // multiplier: 100 + boostPercent (e.g., 110 for 10% boost) + multiplier := new(big.Int).Add(big.NewInt(int64(percentageDivisor)), big.NewInt(int64(boostPercent))) + // gasTipCap = gasTipCap * (100 + boostPercent) / 100 + gasTipCap.Mul(gasTipCap, multiplier).Div(gasTipCap, big.NewInt(int64(percentageDivisor))) + } + + minimumTip := big.NewInt(b.minimumGasTipCap) + if gasTipCap.Cmp(minimumTip) < 0 { + gasTipCap.Set(minimumTip) + } + + latestBlockHeader, err := b.backend.HeaderByNumber(ctx, nil) + if err != nil { + return nil, nil, fmt.Errorf("failed to get latest block header: %w", err) + } + if latestBlockHeader == nil || latestBlockHeader.BaseFee == nil { + return nil, nil, ErrEIP1559NotSupported + } + + // EIP-1559: gasFeeCap = (2 * baseFee) + gasTipCap + gasFeeCap := new(big.Int).Mul(latestBlockHeader.BaseFee, big.NewInt(int64(baseFeeMultiplier))) + gasFeeCap.Add(gasFeeCap, gasTipCap) + + return gasFeeCap, gasTipCap, nil +} diff --git a/pkg/transaction/wrapped/fee_test.go b/pkg/transaction/wrapped/fee_test.go new file mode 100644 index 00000000000..77847ec40d1 --- /dev/null +++ b/pkg/transaction/wrapped/fee_test.go @@ -0,0 +1,128 @@ +// Copyright 2025 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package wrapped_test + +import ( + "context" + "errors" + "math/big" + "testing" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethersphere/bee/v2/pkg/transaction/backendmock" + "github.com/ethersphere/bee/v2/pkg/transaction/wrapped" + "github.com/google/go-cmp/cmp" +) + +func TestSuggestedFeeAndTip(t *testing.T) { + t.Parallel() + + var ( + ctx = context.Background() + minimumGasTipCap = uint64(10) + baseFee = big.NewInt(100) + ) + + testCases := []struct { + name string + gasPrice *big.Int + boostPercent int + mockSuggestGasTip *big.Int + mockSuggestGasErr error + mockHeader *types.Header + mockHeaderErr error + wantGasFeeCap *big.Int + wantGasTipCap *big.Int + wantErr error + }{ + { + name: "with gas price", + gasPrice: big.NewInt(1000), + wantGasFeeCap: big.NewInt(1000), + wantGasTipCap: big.NewInt(1000), + }, + { + name: "suggest tip error", + mockSuggestGasErr: errors.New("suggest tip error"), + wantErr: errors.New("failed to suggest gas tip cap: suggest tip error"), + }, + { + name: "header error", + mockSuggestGasTip: big.NewInt(20), + mockHeaderErr: errors.New("header error"), + wantErr: errors.New("failed to get latest block header: header error"), + }, + { + name: "no base fee", + mockSuggestGasTip: big.NewInt(20), + mockHeader: &types.Header{}, + wantErr: wrapped.ErrEIP1559NotSupported, + }, + { + name: "suggested tip > minimum", + mockSuggestGasTip: big.NewInt(20), + mockHeader: &types.Header{BaseFee: baseFee}, + wantGasFeeCap: big.NewInt(220), // 2*100 + 20 + wantGasTipCap: big.NewInt(20), + }, + { + name: "suggested tip < minimum", + mockSuggestGasTip: big.NewInt(5), + mockHeader: &types.Header{BaseFee: baseFee}, + wantGasFeeCap: big.NewInt(210), // 2*100 + 10 + wantGasTipCap: big.NewInt(10), + }, + { + name: "with boost", + boostPercent: 10, + mockSuggestGasTip: big.NewInt(20), + mockHeader: &types.Header{BaseFee: baseFee}, + wantGasFeeCap: big.NewInt(222), // 2*100 + 22 + wantGasTipCap: big.NewInt(22), // 20 * 1.1 + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + backend := wrapped.NewBackend( + backendmock.New( + backendmock.WithSuggestGasTipCapFunc(func(ctx context.Context) (*big.Int, error) { + return tc.mockSuggestGasTip, tc.mockSuggestGasErr + }), + backendmock.WithHeaderbyNumberFunc(func(ctx context.Context, number *big.Int) (*types.Header, error) { + return tc.mockHeader, tc.mockHeaderErr + }), + ), + minimumGasTipCap, + ) + + gasFeeCap, gasTipCap, err := backend.SuggestedFeeAndTip(ctx, tc.gasPrice, tc.boostPercent) + + if tc.wantErr != nil { + if err == nil { + t.Fatal("expected error but got none") + } + if err.Error() != tc.wantErr.Error() { + t.Fatalf("unexpected error. want %v, got %v", tc.wantErr, err) + } + return + } + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if diff := cmp.Diff(tc.wantGasFeeCap.String(), gasFeeCap.String()); diff != "" { + t.Errorf("gasFeeCap mismatch (-want +got):\n%s", diff) + } + + if diff := cmp.Diff(tc.wantGasTipCap.String(), gasTipCap.String()); diff != "" { + t.Errorf("gasTipCap mismatch (-want +got):\n%s", diff) + } + }) + } +} diff --git a/pkg/transaction/wrapped/metrics.go b/pkg/transaction/wrapped/metrics.go index cb7ce3967c3..a5cfaa554a0 100644 --- a/pkg/transaction/wrapped/metrics.go +++ b/pkg/transaction/wrapped/metrics.go @@ -18,16 +18,14 @@ type metrics struct { BlockNumberCalls prometheus.Counter BlockHeaderCalls prometheus.Counter BalanceCalls prometheus.Counter - CodeAtCalls prometheus.Counter NonceAtCalls prometheus.Counter PendingNonceCalls prometheus.Counter CallContractCalls prometheus.Counter - SuggestGasPriceCalls prometheus.Counter + SuggestGasTipCapCalls prometheus.Counter EstimateGasCalls prometheus.Counter SendTransactionCalls prometheus.Counter FilterLogsCalls prometheus.Counter ChainIDCalls prometheus.Counter - BlockByNumberCalls prometheus.Counter } func newMetrics() metrics { @@ -76,12 +74,6 @@ func newMetrics() metrics { Name: "calls_balance", Help: "Count of eth_getBalance rpc calls", }), - CodeAtCalls: prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: m.Namespace, - Subsystem: subsystem, - Name: "calls_code_at", - Help: "Count of eth_getCode rpc calls", - }), NonceAtCalls: prometheus.NewCounter(prometheus.CounterOpts{ Namespace: m.Namespace, Subsystem: subsystem, @@ -100,11 +92,11 @@ func newMetrics() metrics { Name: "calls_eth_call", Help: "Count of eth_call rpc calls", }), - SuggestGasPriceCalls: prometheus.NewCounter(prometheus.CounterOpts{ + SuggestGasTipCapCalls: prometheus.NewCounter(prometheus.CounterOpts{ Namespace: m.Namespace, Subsystem: subsystem, - Name: "calls_suggest_gasprice", - Help: "Count of eth_suggestGasPrice rpc calls", + Name: "calls_suggest_gas_tip_cap", + Help: "Count of eth_maxPriorityFeePerGas rpc calls", }), EstimateGasCalls: prometheus.NewCounter(prometheus.CounterOpts{ Namespace: m.Namespace, @@ -130,12 +122,6 @@ func newMetrics() metrics { Name: "calls_chain_id", Help: "Count of eth_chainId rpc calls", }), - BlockByNumberCalls: prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: m.Namespace, - Subsystem: subsystem, - Name: "calls_block_by_number", - Help: "Count of eth_getBlockByNumber rpc calls", - }), } } diff --git a/pkg/transaction/wrapped/wrapped.go b/pkg/transaction/wrapped/wrapped.go index 87ec74da7e9..1a51a618a88 100644 --- a/pkg/transaction/wrapped/wrapped.go +++ b/pkg/transaction/wrapped/wrapped.go @@ -12,21 +12,25 @@ import ( "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethclient" "github.com/ethersphere/bee/v2/pkg/transaction" + "github.com/ethersphere/bee/v2/pkg/transaction/backend" ) -var _ transaction.Backend = (*wrappedBackend)(nil) +var ( + _ transaction.Backend = (*wrappedBackend)(nil) +) type wrappedBackend struct { - backend *ethclient.Client - metrics metrics + backend backend.Geth + metrics metrics + minimumGasTipCap int64 } -func NewBackend(backend *ethclient.Client) transaction.Backend { +func NewBackend(backend backend.Geth, minimumGasTipCap uint64) transaction.Backend { return &wrappedBackend{ - backend: backend, - metrics: newMetrics(), + backend: backend, + minimumGasTipCap: int64(minimumGasTipCap), + metrics: newMetrics(), } } @@ -102,17 +106,6 @@ func (b *wrappedBackend) NonceAt(ctx context.Context, account common.Address, bl return nonce, nil } -func (b *wrappedBackend) CodeAt(ctx context.Context, contract common.Address, blockNumber *big.Int) ([]byte, error) { - b.metrics.TotalRPCCalls.Inc() - b.metrics.CodeAtCalls.Inc() - code, err := b.backend.CodeAt(ctx, contract, blockNumber) - if err != nil { - b.metrics.TotalRPCErrors.Inc() - return nil, err - } - return code, nil -} - func (b *wrappedBackend) CallContract(ctx context.Context, call ethereum.CallMsg, blockNumber *big.Int) ([]byte, error) { b.metrics.TotalRPCCalls.Inc() b.metrics.CallContractCalls.Inc() @@ -135,20 +128,9 @@ func (b *wrappedBackend) PendingNonceAt(ctx context.Context, account common.Addr return nonce, nil } -func (b *wrappedBackend) SuggestGasPrice(ctx context.Context) (*big.Int, error) { - b.metrics.TotalRPCCalls.Inc() - b.metrics.SuggestGasPriceCalls.Inc() - gasPrice, err := b.backend.SuggestGasPrice(ctx) - if err != nil { - b.metrics.TotalRPCErrors.Inc() - return nil, err - } - return gasPrice, nil -} - func (b *wrappedBackend) SuggestGasTipCap(ctx context.Context) (*big.Int, error) { b.metrics.TotalRPCCalls.Inc() - b.metrics.SuggestGasPriceCalls.Inc() + b.metrics.SuggestGasTipCapCalls.Inc() gasTipCap, err := b.backend.SuggestGasTipCap(ctx) if err != nil { b.metrics.TotalRPCErrors.Inc() @@ -201,21 +183,6 @@ func (b *wrappedBackend) ChainID(ctx context.Context) (*big.Int, error) { return chainID, nil } -// BlockByNumber implements transaction.Backend. -func (b *wrappedBackend) BlockByNumber(ctx context.Context, number *big.Int) (*types.Block, error) { - b.metrics.TotalRPCCalls.Inc() - b.metrics.BlockByNumberCalls.Inc() - block, err := b.backend.BlockByNumber(ctx, number) - if err != nil { - if !errors.Is(err, ethereum.NotFound) { - b.metrics.TotalRPCErrors.Inc() - } - return nil, err - } - return block, nil -} - -func (b *wrappedBackend) Close() error { +func (b *wrappedBackend) Close() { b.backend.Close() - return nil } From 370d1888c3cd5756b6216a9dcedec96fad6b2146 Mon Sep 17 00:00:00 2001 From: tanhuaan Date: Mon, 11 Aug 2025 01:33:17 +0800 Subject: [PATCH 09/96] chore: make function comments match function names (#5183) Signed-off-by: tanhuaan --- pkg/file/pipeline/encryption/encryption_test.go | 2 +- pkg/file/pipeline/feeder/feeder.go | 2 +- pkg/pusher/pusher_test.go | 2 +- pkg/pushsync/pushsync_test.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/file/pipeline/encryption/encryption_test.go b/pkg/file/pipeline/encryption/encryption_test.go index edcfa33a46f..2010f6b8327 100644 --- a/pkg/file/pipeline/encryption/encryption_test.go +++ b/pkg/file/pipeline/encryption/encryption_test.go @@ -38,7 +38,7 @@ func init() { } } -// TestEncyrption tests that the encryption writer works correctly. +// TestEncryption tests that the encryption writer works correctly. func TestEncryption(t *testing.T) { t.Parallel() diff --git a/pkg/file/pipeline/feeder/feeder.go b/pkg/file/pipeline/feeder/feeder.go index 3d84f597d44..6ba722796f2 100644 --- a/pkg/file/pipeline/feeder/feeder.go +++ b/pkg/file/pipeline/feeder/feeder.go @@ -21,7 +21,7 @@ type chunkFeeder struct { wrote int64 } -// newChunkFeederWriter creates a new chunkFeeder that allows for partial +// NewChunkFeederWriter creates a new chunkFeeder that allows for partial // writes into the pipeline. Any pending data in the buffer is flushed to // subsequent writers when Sum() is called. func NewChunkFeederWriter(size int, next pipeline.ChainWriter) pipeline.Interface { diff --git a/pkg/pusher/pusher_test.go b/pkg/pusher/pusher_test.go index d45101d06f5..4abe0d67537 100644 --- a/pkg/pusher/pusher_test.go +++ b/pkg/pusher/pusher_test.go @@ -109,7 +109,7 @@ func (m *mockStorer) ReservePutter() storage.Putter { ) } -// TestSendChunkToPushSync sends a chunk to pushsync to be sent to its closest peer and get a receipt. +// TestChunkSyncing sends a chunk to pushsync to be sent to its closest peer and get a receipt. // once the receipt is got this check to see if the localstore is updated to see if the chunk is set // as ModeSetSync status. func TestChunkSyncing(t *testing.T) { diff --git a/pkg/pushsync/pushsync_test.go b/pkg/pushsync/pushsync_test.go index 7e1c661f089..90e6d654a5b 100644 --- a/pkg/pushsync/pushsync_test.go +++ b/pkg/pushsync/pushsync_test.go @@ -308,7 +308,7 @@ func TestShallowReceiptTolerance(t *testing.T) { waitOnRecordAndTest(t, closestPeer, recorder, chunk.Address(), nil) } -// PushChunkToClosest tests the sending of chunk to closest peer from the origination source perspective. +// TestPushChunkToClosest tests the sending of chunk to closest peer from the origination source perspective. // it also checks whether the tags are incremented properly if they are present func TestPushChunkToClosest(t *testing.T) { t.Parallel() From e7242df19061195925e5dea82f48aae8a58b658a Mon Sep 17 00:00:00 2001 From: youzichuan Date: Wed, 13 Aug 2025 18:49:39 +0800 Subject: [PATCH 10/96] refactor: replace context.WithCancel with t.Context (#5188) Signed-off-by: youzichuan --- pkg/file/joiner/joiner_test.go | 27 ++++++---------- pkg/file/splitter/internal/job_test.go | 3 +- pkg/p2p/libp2p/connections_test.go | 45 +++++++++----------------- pkg/p2p/libp2p/headers_test.go | 9 ++---- pkg/p2p/libp2p/protocols_test.go | 24 +++++--------- pkg/p2p/libp2p/tracing_test.go | 6 ++-- 6 files changed, 38 insertions(+), 76 deletions(-) diff --git a/pkg/file/joiner/joiner_test.go b/pkg/file/joiner/joiner_test.go index dfed2aecc91..27591252548 100644 --- a/pkg/file/joiner/joiner_test.go +++ b/pkg/file/joiner/joiner_test.go @@ -56,8 +56,7 @@ func TestJoinerSingleChunk(t *testing.T) { store := inmemchunkstore.New() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() // create the chunk to mockAddrHex := fmt.Sprintf("%064s", "2a") @@ -95,8 +94,7 @@ func TestJoinerDecryptingStore_NormalChunk(t *testing.T) { st := inmemchunkstore.New() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() // create the chunk to mockAddrHex := fmt.Sprintf("%064s", "2a") @@ -134,8 +132,7 @@ func TestJoinerWithReference(t *testing.T) { st := inmemchunkstore.New() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() // create root chunk and two data chunks referenced in the root chunk rootChunk := filetest.GenerateTestRandomFileChunk(swarm.ZeroAddress, swarm.ChunkSize*2, swarm.SectionSize*2) @@ -185,8 +182,7 @@ func TestJoinerMalformed(t *testing.T) { store := inmemchunkstore.New() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() subTrie := []byte{8085: 1} pb := builder.NewPipelineBuilder(ctx, store, false, 0) @@ -253,8 +249,7 @@ func TestEncryptDecrypt(t *testing.T) { if err != nil { t.Fatal(err) } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() pipe := builder.NewPipelineBuilder(ctx, store, true, 0) testDataReader := bytes.NewReader(testData) resultAddress, err := builder.FeedPipeline(ctx, pipe, testDataReader) @@ -335,8 +330,7 @@ func TestSeek(t *testing.T) { t.Run(tc.name, func(t *testing.T) { t.Parallel() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() store := inmemchunkstore.New() testutil.CleanupCloser(t, store) @@ -612,8 +606,7 @@ func TestPrefetch(t *testing.T) { t.Run(tc.name, func(t *testing.T) { t.Parallel() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() store := inmemchunkstore.New() testutil.CleanupCloser(t, store) @@ -917,8 +910,7 @@ func TestJoinerIterateChunkAddresses_Encrypted(t *testing.T) { if err != nil { t.Fatal(err) } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() pipe := builder.NewPipelineBuilder(ctx, store, true, 0) testDataReader := bytes.NewReader(testData) resultAddress, err := builder.FeedPipeline(ctx, pipe, testDataReader) @@ -1072,8 +1064,7 @@ func TestJoinerRedundancy(t *testing.T) { }, } { t.Run(fmt.Sprintf("redundancy=%d encryption=%t", tc.rLevel, tc.encryptChunk), func(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() shardCnt := tc.rLevel.GetMaxShards() parityCnt := tc.rLevel.GetParities(shardCnt) if tc.encryptChunk { diff --git a/pkg/file/splitter/internal/job_test.go b/pkg/file/splitter/internal/job_test.go index bc7945c4b42..7e9824d4748 100644 --- a/pkg/file/splitter/internal/job_test.go +++ b/pkg/file/splitter/internal/job_test.go @@ -29,8 +29,7 @@ func TestSplitterJobPartialSingleChunk(t *testing.T) { store := inmemchunkstore.New() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() data := []byte("foo") j := internal.NewSimpleSplitterJob(ctx, store, int64(len(data)), false) diff --git a/pkg/p2p/libp2p/connections_test.go b/pkg/p2p/libp2p/connections_test.go index c85b016d801..51227ab1555 100644 --- a/pkg/p2p/libp2p/connections_test.go +++ b/pkg/p2p/libp2p/connections_test.go @@ -59,8 +59,7 @@ func TestAddresses(t *testing.T) { func TestConnectDisconnect(t *testing.T) { t.Parallel() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() s1, overlay1 := newService(t, 1, libp2pServiceOpts{libp2pOpts: libp2p.Options{ FullNode: true, @@ -88,8 +87,7 @@ func TestConnectDisconnect(t *testing.T) { func TestConnectToLightPeer(t *testing.T) { t.Parallel() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() s1, _ := newService(t, 1, libp2pServiceOpts{libp2pOpts: libp2p.Options{ FullNode: false, @@ -110,8 +108,7 @@ func TestConnectToLightPeer(t *testing.T) { func TestLightPeerLimit(t *testing.T) { t.Parallel() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() var ( limit = 3 @@ -160,8 +157,7 @@ func TestStreamsMaxIncomingLimit(t *testing.T) { maxIncomingStreams := 5000 - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() s1, overlay1 := newService(t, 1, libp2pServiceOpts{libp2pOpts: libp2p.Options{ FullNode: true, @@ -283,8 +279,7 @@ func TestStreamsMaxIncomingLimit(t *testing.T) { func TestDoubleConnect(t *testing.T) { t.Parallel() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() s1, overlay1 := newService(t, 1, libp2pServiceOpts{libp2pOpts: libp2p.Options{ FullNode: true, @@ -311,8 +306,7 @@ func TestDoubleConnect(t *testing.T) { func TestDoubleDisconnect(t *testing.T) { t.Parallel() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() s1, overlay1 := newService(t, 1, libp2pServiceOpts{libp2pOpts: libp2p.Options{ FullNode: true, @@ -347,8 +341,7 @@ func TestDoubleDisconnect(t *testing.T) { func TestMultipleConnectDisconnect(t *testing.T) { t.Parallel() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() s1, overlay1 := newService(t, 1, libp2pServiceOpts{libp2pOpts: libp2p.Options{ FullNode: true, @@ -392,8 +385,7 @@ func TestMultipleConnectDisconnect(t *testing.T) { func TestConnectDisconnectOnAllAddresses(t *testing.T) { t.Parallel() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() s1, overlay1 := newService(t, 1, libp2pServiceOpts{libp2pOpts: libp2p.Options{ FullNode: true, @@ -426,8 +418,7 @@ func TestConnectDisconnectOnAllAddresses(t *testing.T) { func TestDoubleConnectOnAllAddresses(t *testing.T) { t.Parallel() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() s1, overlay1 := newService(t, 1, libp2pServiceOpts{ notifier: mockNotifier(noopCf, noopDf, true), @@ -471,8 +462,7 @@ func TestDoubleConnectOnAllAddresses(t *testing.T) { func TestDifferentNetworkIDs(t *testing.T) { t.Parallel() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() s1, _ := newService(t, 1, libp2pServiceOpts{}) s2, _ := newService(t, 2, libp2pServiceOpts{}) @@ -490,8 +480,7 @@ func TestDifferentNetworkIDs(t *testing.T) { func TestConnectWithEnabledWSTransports(t *testing.T) { t.Parallel() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() s1, overlay1 := newService(t, 1, libp2pServiceOpts{ libp2pOpts: libp2p.Options{ @@ -521,8 +510,7 @@ func TestConnectWithEnabledWSTransports(t *testing.T) { func TestConnectRepeatHandshake(t *testing.T) { t.Parallel() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() s1, overlay1 := newService(t, 1, libp2pServiceOpts{libp2pOpts: libp2p.Options{ FullNode: true, @@ -970,8 +958,7 @@ func TestWithDisconnectStreams(t *testing.T) { const headersRWTimeout = 60 * time.Second - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() s1, overlay1 := newService(t, 1, libp2pServiceOpts{libp2pOpts: libp2p.Options{ FullNode: true, @@ -1021,8 +1008,7 @@ func TestWithDisconnectStreams(t *testing.T) { func TestWithBlocklistStreams(t *testing.T) { t.Parallel() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() s1, overlay1 := newService(t, 1, libp2pServiceOpts{libp2pOpts: libp2p.Options{ FullNode: true, @@ -1073,8 +1059,7 @@ func TestWithBlocklistStreams(t *testing.T) { func TestUserAgentLogging(t *testing.T) { t.Parallel() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() // use concurrent-safe buffers as handlers are logging concurrently s1Logs := new(buffer) diff --git a/pkg/p2p/libp2p/headers_test.go b/pkg/p2p/libp2p/headers_test.go index 5805dafa0d4..d46653bd719 100644 --- a/pkg/p2p/libp2p/headers_test.go +++ b/pkg/p2p/libp2p/headers_test.go @@ -23,8 +23,7 @@ func TestHeaders(t *testing.T) { "other-key": []byte("other-value"), } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() s1, overlay1 := newService(t, 1, libp2pServiceOpts{libp2pOpts: libp2p.Options{ FullNode: true, @@ -74,8 +73,7 @@ func TestHeaders(t *testing.T) { func TestHeaders_empty(t *testing.T) { t.Parallel() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() s1, overlay1 := newService(t, 1, libp2pServiceOpts{libp2pOpts: libp2p.Options{ FullNode: true, @@ -134,8 +132,7 @@ func TestHeadler(t *testing.T) { "other-sent-key": []byte("other-sent-value"), } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() s1, overlay1 := newService(t, 1, libp2pServiceOpts{libp2pOpts: libp2p.Options{ FullNode: true, diff --git a/pkg/p2p/libp2p/protocols_test.go b/pkg/p2p/libp2p/protocols_test.go index 4563083726a..50a9d9c2970 100644 --- a/pkg/p2p/libp2p/protocols_test.go +++ b/pkg/p2p/libp2p/protocols_test.go @@ -26,8 +26,7 @@ import ( func TestNewStream(t *testing.T) { t.Parallel() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() s1, overlay1 := newService(t, 1, libp2pServiceOpts{libp2pOpts: libp2p.Options{ FullNode: true, @@ -61,8 +60,7 @@ func TestNewStream(t *testing.T) { func TestNewStream_OnlyFull(t *testing.T) { t.Parallel() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() s1, overlay1 := newService(t, 1, libp2pServiceOpts{libp2pOpts: libp2p.Options{ FullNode: true, @@ -101,8 +99,7 @@ func TestNewStream_OnlyFull(t *testing.T) { func TestNewStream_Mixed(t *testing.T) { t.Parallel() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() s1, overlay1 := newService(t, 1, libp2pServiceOpts{libp2pOpts: libp2p.Options{ FullNode: true, @@ -140,8 +137,7 @@ func TestNewStream_Mixed(t *testing.T) { func TestNewStreamMulti(t *testing.T) { t.Parallel() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() s1, overlay1 := newService(t, 1, libp2pServiceOpts{libp2pOpts: libp2p.Options{ FullNode: true, @@ -190,8 +186,7 @@ func TestNewStreamMulti(t *testing.T) { func TestNewStream_errNotSupported(t *testing.T) { t.Parallel() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() s1, overlay1 := newService(t, 1, libp2pServiceOpts{libp2pOpts: libp2p.Options{ FullNode: true, @@ -229,8 +224,7 @@ func TestNewStream_errNotSupported(t *testing.T) { func TestNewStream_semanticVersioning(t *testing.T) { t.Parallel() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() s1, overlay1 := newService(t, 1, libp2pServiceOpts{libp2pOpts: libp2p.Options{ FullNode: true, @@ -292,8 +286,7 @@ func TestNewStream_semanticVersioning(t *testing.T) { func TestDisconnectError(t *testing.T) { t.Parallel() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() s1, overlay1 := newService(t, 1, libp2pServiceOpts{libp2pOpts: libp2p.Options{ FullNode: true, @@ -324,8 +317,7 @@ func TestDisconnectError(t *testing.T) { func TestConnectDisconnectEvents(t *testing.T) { t.Parallel() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() s1, overlay1 := newService(t, 1, libp2pServiceOpts{libp2pOpts: libp2p.Options{ FullNode: true, diff --git a/pkg/p2p/libp2p/tracing_test.go b/pkg/p2p/libp2p/tracing_test.go index 7e02f8e2d7c..533a49088ec 100644 --- a/pkg/p2p/libp2p/tracing_test.go +++ b/pkg/p2p/libp2p/tracing_test.go @@ -58,15 +58,13 @@ func TestTracing(t *testing.T) { addr := serviceUnderlayAddress(t, s1) - connectContext, connectCancel := context.WithCancel(context.Background()) - defer connectCancel() + connectContext := t.Context() if _, err := s2.Connect(connectContext, addr); err != nil { t.Fatal(err) } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() span, _, ctx := tracer2.StartSpanFromContext(ctx, "test-p2p-client", nil) defer span.Finish() From 3a5de30aba477560bfc503632479f4793d68dcef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ljubi=C5=A1a=20Ga=C4=8Devi=C4=87?= <35105035+gacevicljubisa@users.noreply.github.com> Date: Fri, 15 Aug 2025 13:51:54 +0200 Subject: [PATCH 11/96] chore(ci): bump k3s to v1.31.10+k3s1 (#5186) --- .github/workflows/beekeeper.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/beekeeper.yml b/.github/workflows/beekeeper.yml index ff36eff2514..5006a520d2c 100644 --- a/.github/workflows/beekeeper.yml +++ b/.github/workflows/beekeeper.yml @@ -8,7 +8,7 @@ on: - "**" env: - K3S_VERSION: "v1.30.3+k3s1" + K3S_VERSION: "v1.31.10+k3s1" REPLICA: 3 RUN_TYPE: "PR RUN" SETUP_CONTRACT_IMAGE: "ethersphere/bee-localchain" @@ -190,7 +190,6 @@ jobs: if ${{ steps.pushsync-chunks-2.outcome=='failure' }}; then FAILED=pushsync-chunks-2; fi if ${{ steps.retrieval.outcome=='failure' }}; then FAILED=retrieval; fi if ${{ steps.manifest.outcome=='failure' }}; then FAILED=manifest; fi - if ${{ steps.content-availability.outcome=='failure' }}; then FAILED=content-availability; fi if ${{ steps.act.outcome=='failure' }}; then FAILED=act; fi curl -sSf -X POST -H "Content-Type: application/json" -d "{\"text\": \"**${RUN_TYPE}** Beekeeper Error\nBranch: \`${{ github.head_ref }}\`\nUser: @${{ github.event.pull_request.user.login }}\nDebugging artifacts: [click](https://$BUCKET_NAME.$AWS_ENDPOINT/artifacts_$VERTAG.tar.gz)\nStep failed: \`${FAILED}\`\"}" https://beehive.ethswarm.org/hooks/${{ secrets.TUNSHELL_KEY }} echo "Failed test: ${FAILED}" From 2378edd56fc341878b96c0bdb8865a30e025f230 Mon Sep 17 00:00:00 2001 From: nugaon Date: Thu, 21 Aug 2025 16:00:12 +0200 Subject: [PATCH 12/96] feat: bootup time metrics (#5162) --- pkg/node/metrics.go | 46 ++++++++++++++++++++++++++++++++++++++++ pkg/node/node.go | 50 ++++++++++++++++++++++++++++++++++++++++++-- pkg/salud/metrics.go | 15 +++++++++++++ pkg/salud/salud.go | 33 +++++++++++++++++++++++------ 4 files changed, 136 insertions(+), 8 deletions(-) create mode 100644 pkg/node/metrics.go diff --git a/pkg/node/metrics.go b/pkg/node/metrics.go new file mode 100644 index 00000000000..90005ceec7a --- /dev/null +++ b/pkg/node/metrics.go @@ -0,0 +1,46 @@ +// Copyright 2022 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package node + +import ( + "github.com/ethersphere/bee/v2/pkg/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +type nodeMetrics struct { + // WarmupDuration measures time in seconds for the node warmup to complete + WarmupDuration prometheus.Histogram + // FullSyncDuration measures time in seconds for the full sync to complete + FullSyncDuration prometheus.Histogram +} + +func newMetrics() nodeMetrics { + subsystem := "init" + + return nodeMetrics{ + WarmupDuration: prometheus.NewHistogram( + prometheus.HistogramOpts{ + Namespace: metrics.Namespace, + Subsystem: subsystem, + Name: "warmup_duration_seconds", + Help: "Duration in seconds for node warmup to complete", + }, + ), + FullSyncDuration: prometheus.NewHistogram( + prometheus.HistogramOpts{ + Namespace: metrics.Namespace, + Subsystem: subsystem, + Name: "full_sync_duration_seconds", + Help: "Duration in seconds for node warmup to complete", + }, + ), + } +} + +// RegisterMetrics registers all metrics from the package +func (m nodeMetrics) RegisterMetrics() { + prometheus.MustRegister(m.WarmupDuration) + prometheus.MustRegister(m.FullSyncDuration) +} diff --git a/pkg/node/node.go b/pkg/node/node.go index 477fe7c230d..f3f37b3f054 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -207,6 +207,13 @@ func NewBee( session accesscontrol.Session, o *Options, ) (b *Bee, err error) { + // start time for node warmup duration measurement + warmupStartTime := time.Now() + var pullSyncStartTime time.Time + + nodeMetrics := newMetrics() + nodeMetrics.RegisterMetrics() + tracer, tracerCloser, err := tracing.NewTracer(&tracing.Options{ Enabled: o.TracingEnabled, Endpoint: o.TracingEndpoint, @@ -598,9 +605,18 @@ func NewBee( logger.Info("node warmup check initiated. monitoring activity rate to determine readiness.", "startTime", t) } - detector.OnStabilized = func(t time.Time, totalCount int) { - logger.Info("node warmup complete. system is considered stable and ready.", "stabilizationTime", t, "totalMonitoredEvents", totalCount) + warmupMeasurement := func(t time.Time, totalCount int) { + warmupDuration := t.Sub(warmupStartTime).Seconds() + logger.Info("node warmup complete. system is considered stable and ready.", + "stabilizationTime", t, + "totalMonitoredEvents", totalCount, + "warmupDurationSeconds", warmupDuration) + + // Record the warmup duration in the prometheus metric + nodeMetrics.WarmupDuration.Observe(warmupDuration) + pullSyncStartTime = t } + detector.OnStabilized = warmupMeasurement detector.OnPeriodComplete = func(t time.Time, periodCount int, stDev float64) { logger.Debug("node warmup check: period complete.", "periodEndTime", t, "eventsInPeriod", periodCount, "rateStdDev", stDev) @@ -1133,6 +1149,36 @@ func NewBee( localStore.StartReserveWorker(ctx, pullerService, waitNetworkRFunc) nodeStatus.SetSync(pullerService) + // measure full sync duration + detector.OnStabilized = func(t time.Time, totalCount int) { + warmupMeasurement(t, totalCount) + + reserveTreshold := reserveCapacity >> 1 + isFullySynced := func() bool { + return pullerService.SyncRate() == 0 && saludService.IsHealthy() && localStore.ReserveSize() >= reserveTreshold + } + + syncCheckTicker := time.NewTicker(2 * time.Second) + go func() { + defer syncCheckTicker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-syncCheckTicker.C: + synced := isFullySynced() + logger.Debug("sync status check", "synced", synced, "reserveSize", localStore.ReserveSize(), "threshold", reserveTreshold, "syncRate", pullerService.SyncRate()) + if synced { + fullSyncTime := pullSyncStartTime.Sub(t) + nodeMetrics.FullSyncDuration.Observe(fullSyncTime.Seconds()) + syncCheckTicker.Stop() + return + } + } + } + }() + } + if o.EnableStorageIncentives { redistributionContractAddress := chainCfg.RedistributionAddress diff --git a/pkg/salud/metrics.go b/pkg/salud/metrics.go index a99087d219b..8ac5abdb517 100644 --- a/pkg/salud/metrics.go +++ b/pkg/salud/metrics.go @@ -19,6 +19,8 @@ type metrics struct { ReserveSizePercentErr prometheus.Gauge Healthy prometheus.Counter Unhealthy prometheus.Counter + NeighborhoodAvgDur prometheus.Gauge + NeighborCount prometheus.Gauge } func newMetrics() metrics { @@ -79,6 +81,19 @@ func newMetrics() metrics { Name: "reserve_size_percentage_err", Help: "Percentage error of the reservesize relative to the network average.", }), + // Neighborhood-specific metrics + NeighborhoodAvgDur: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: m.Namespace, + Subsystem: subsystem, + Name: "neighborhood_dur", + Help: "Average duration for snapshot response from neighborhood peers.", + }), + NeighborCount: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: m.Namespace, + Subsystem: subsystem, + Name: "neighbors", + Help: "Number of neighborhood peers.", + }), } } diff --git a/pkg/salud/salud.go b/pkg/salud/salud.go index dc931f86081..28fe648d030 100644 --- a/pkg/salud/salud.go +++ b/pkg/salud/salud.go @@ -134,10 +134,12 @@ type peer struct { // the allowed thresholds. func (s *service) salud(mode string, durPercentile float64, connsPercentile float64) { var ( - mtx sync.Mutex - wg sync.WaitGroup - totaldur float64 - peers []peer + mtx sync.Mutex + wg sync.WaitGroup + totaldur float64 + peers []peer + neighborhoodPeers uint + neighborhoodTotalDur float64 ) err := s.topology.EachConnectedPeer(func(addr swarm.Address, bin uint8) (stop bool, jumpToNext bool, err error) { @@ -163,7 +165,12 @@ func (s *service) salud(mode string, durPercentile float64, connsPercentile floa mtx.Lock() totaldur += dur.Seconds() - peers = append(peers, peer{snapshot, dur, addr, bin, s.reserve.IsWithinStorageRadius(addr)}) + peer := peer{snapshot, dur, addr, bin, s.reserve.IsWithinStorageRadius(addr)} + peers = append(peers, peer) + if peer.neighbor { + neighborhoodPeers++ + neighborhoodTotalDur += dur.Seconds() + } mtx.Unlock() }() return false, false, nil @@ -184,6 +191,20 @@ func (s *service) salud(mode string, durPercentile float64, connsPercentile floa pConns := percentileConns(peers, connsPercentile) commitment := commitment(peers) + if neighborhoodPeers > 0 { + neighborhoodAvgDur := neighborhoodTotalDur / float64(neighborhoodPeers) + + s.metrics.NeighborhoodAvgDur.Set(neighborhoodAvgDur) + s.metrics.NeighborCount.Set(float64(neighborhoodPeers)) + + s.logger.Debug("neighborhood metrics", "avg_dur", neighborhoodAvgDur, "count", neighborhoodPeers) + } else { + s.metrics.NeighborhoodAvgDur.Set(0) + s.metrics.NeighborCount.Set(0) + + s.logger.Debug("no neighborhood peers found for metrics") + } + s.metrics.AvgDur.Set(avgDur) s.metrics.PDur.Set(pDur) s.metrics.PConns.Set(float64(pConns)) @@ -191,7 +212,7 @@ func (s *service) salud(mode string, durPercentile float64, connsPercentile floa s.metrics.NeighborhoodRadius.Set(float64(nHoodRadius)) s.metrics.Commitment.Set(float64(commitment)) - s.logger.Debug("computed", "avg_dur", avgDur, "pDur", pDur, "pConns", pConns, "network_radius", networkRadius, "neighborhood_radius", nHoodRadius, "batch_commitment", commitment) + s.logger.Debug("computed", "avg_dur", avgDur, "pDur", pDur, "pConns", pConns, "network_radius", networkRadius, "neighborhood_radius", nHoodRadius, "batch_commitment", commitment, "neighborhood_peers", neighborhoodPeers) // sort peers by duration, highest first to give priority to the fastest peers sort.Slice(peers, func(i, j int) bool { From 29ea2b7eafcb6c7bb61fda9f0b7ec70deafcc56e Mon Sep 17 00:00:00 2001 From: nugaon Date: Tue, 2 Sep 2025 11:41:32 +0200 Subject: [PATCH 13/96] fix: register init metrics (#5196) --- pkg/node/metrics.go | 6 ++---- pkg/node/node.go | 2 +- pkg/storageincentives/soc_mine_test.go | 13 ++++++++----- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/pkg/node/metrics.go b/pkg/node/metrics.go index 90005ceec7a..959363eefb9 100644 --- a/pkg/node/metrics.go +++ b/pkg/node/metrics.go @@ -39,8 +39,6 @@ func newMetrics() nodeMetrics { } } -// RegisterMetrics registers all metrics from the package -func (m nodeMetrics) RegisterMetrics() { - prometheus.MustRegister(m.WarmupDuration) - prometheus.MustRegister(m.FullSyncDuration) +func getMetrics(nodeMetrics nodeMetrics) []prometheus.Collector { + return metrics.PrometheusCollectorsFromFields(nodeMetrics) } diff --git a/pkg/node/node.go b/pkg/node/node.go index f3f37b3f054..226ac273c5e 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -212,7 +212,6 @@ func NewBee( var pullSyncStartTime time.Time nodeMetrics := newMetrics() - nodeMetrics.RegisterMetrics() tracer, tracerCloser, err := tracing.NewTracer(&tracing.Options{ Enabled: o.TracingEnabled, @@ -1266,6 +1265,7 @@ func NewBee( apiService.MustRegisterMetrics(kad.Metrics()...) apiService.MustRegisterMetrics(saludService.Metrics()...) apiService.MustRegisterMetrics(stateStoreMetrics.Metrics()...) + apiService.MustRegisterMetrics(getMetrics(nodeMetrics)...) if pullerService != nil { apiService.MustRegisterMetrics(pullerService.Metrics()...) diff --git a/pkg/storageincentives/soc_mine_test.go b/pkg/storageincentives/soc_mine_test.go index 29a8b5e0898..0e3a5090f47 100644 --- a/pkg/storageincentives/soc_mine_test.go +++ b/pkg/storageincentives/soc_mine_test.go @@ -40,7 +40,8 @@ func TestSocMine(t *testing.T) { } // the transformed address hasher factory function prefixhasher := func() hash.Hash { return swarm.NewPrefixHasher(prefix) } - trHasher := func() hash.Hash { return bmt.NewHasher(prefixhasher) } + // Create a pool for efficient hasher reuse + trHasherPool := bmt.NewPool(bmt.NewConf(prefixhasher, swarm.BmtBranches, 8)) // the bignum cast of the maximum sample value (upper bound on transformed addresses as a 256-bit article) // this constant is for a minimum reserve size of 2 million chunks with sample size of 16 // = 1.284401 * 10^71 = 1284401 + 66 0-s @@ -79,13 +80,13 @@ func TestSocMine(t *testing.T) { // for sanity check: given a filterSOCAddr requiring a 0 leading bit (chance of 1/2) // we expect an overall rough 4 million chunks to be mined to create this sample // for 8 workers that is half a million round on average per worker - err = makeChunks(t, signer, sampleSize, filterSOCAddr, filterTrAddr, trHasher) + err = makeChunks(t, signer, sampleSize, filterSOCAddr, filterTrAddr, trHasherPool) if err != nil { t.Fatal(err) } } -func makeChunks(t *testing.T, signer crypto.Signer, sampleSize int, filterSOCAddr func(swarm.Address) bool, filterTrAddr func(swarm.Address) (bool, error), trHasher func() hash.Hash) error { +func makeChunks(t *testing.T, signer crypto.Signer, sampleSize int, filterSOCAddr func(swarm.Address) bool, filterTrAddr func(swarm.Address) (bool, error), trHasherPool *bmt.Pool) error { t.Helper() // set owner address from signer @@ -129,6 +130,8 @@ func makeChunks(t *testing.T, signer crypto.Signer, sampleSize int, filterSOCAdd eg.Go(func() (err error) { offset := i * 4 found := 0 + // Get one hasher per goroutine from the pool to avoid race conditions + hasher := trHasherPool.Get() for seed := uint32(1); ; seed++ { select { case <-ectx.Done(): @@ -148,9 +151,9 @@ func makeChunks(t *testing.T, signer crypto.Signer, sampleSize int, filterSOCAdd if !filterSOCAddr(addr) { continue } - hasher := trHasher() data := s.WrappedChunk().Data() - hasher.(*bmt.Hasher).SetHeader(data[:8]) + hasher.Reset() + hasher.SetHeader(data[:8]) _, err = hasher.Write(data[8:]) if err != nil { return err From 0e8594d1dcccd46cd1a72ca4af1c3e02db055614 Mon Sep 17 00:00:00 2001 From: nugaon Date: Wed, 3 Sep 2025 08:56:46 +0200 Subject: [PATCH 14/96] refactor: init metrics (#5198) --- pkg/node/metrics.go | 10 ++++++++-- pkg/node/node.go | 6 +++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/pkg/node/metrics.go b/pkg/node/metrics.go index 959363eefb9..a161c39ffe7 100644 --- a/pkg/node/metrics.go +++ b/pkg/node/metrics.go @@ -26,14 +26,20 @@ func newMetrics() nodeMetrics { Subsystem: subsystem, Name: "warmup_duration_seconds", Help: "Duration in seconds for node warmup to complete", + // middle range should be more infrequent (because of addressbook) + Buckets: []float64{10, 20, 25, 30, 35, 40, 45, 50, 60, 70, 90, 120, 180, 240, 300, 350, 380, 400, 420, 440, 460, 480, 550, 600}, }, ), FullSyncDuration: prometheus.NewHistogram( prometheus.HistogramOpts{ Namespace: metrics.Namespace, Subsystem: subsystem, - Name: "full_sync_duration_seconds", - Help: "Duration in seconds for node warmup to complete", + Name: "full_sync_duration_minutes", + Help: "Duration in minutes for node full sync to complete", + // middle range should be more frequent + Buckets: []float64{80, 90, 100, 110, + 120, 125, 130, 135, 140, 145, 150, 155, 160, 165, 170, 175, 180, // 2-3 hours range + 190, 200, 210, 220, 230, 240}, }, ), } diff --git a/pkg/node/node.go b/pkg/node/node.go index 226ac273c5e..67d185a55c5 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -611,7 +611,6 @@ func NewBee( "totalMonitoredEvents", totalCount, "warmupDurationSeconds", warmupDuration) - // Record the warmup duration in the prometheus metric nodeMetrics.WarmupDuration.Observe(warmupDuration) pullSyncStartTime = t } @@ -1166,10 +1165,11 @@ func NewBee( return case <-syncCheckTicker.C: synced := isFullySynced() - logger.Debug("sync status check", "synced", synced, "reserveSize", localStore.ReserveSize(), "threshold", reserveTreshold, "syncRate", pullerService.SyncRate()) + logger.Debug("sync status check", "synced", synced, "reserveSize", localStore.ReserveSize(), "syncRate", pullerService.SyncRate()) if synced { fullSyncTime := pullSyncStartTime.Sub(t) - nodeMetrics.FullSyncDuration.Observe(fullSyncTime.Seconds()) + logger.Info("full sync duration", "duration", fullSyncTime) + nodeMetrics.FullSyncDuration.Observe(fullSyncTime.Minutes()) syncCheckTicker.Stop() return } From e0cbd67a09eb7458644a2cf66f95ffd91184397e Mon Sep 17 00:00:00 2001 From: pxwanglu Date: Tue, 9 Sep 2025 15:35:37 +0800 Subject: [PATCH 15/96] chore: fix some function names in comment (#5195) Signed-off-by: pxwanglu --- pkg/crypto/crypto.go | 4 ++-- pkg/file/pipeline/hashtrie/hashtrie.go | 2 +- pkg/p2p/libp2p/peer.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/crypto/crypto.go b/pkg/crypto/crypto.go index c5ac2c7128c..6c43a1e4528 100644 --- a/pkg/crypto/crypto.go +++ b/pkg/crypto/crypto.go @@ -92,12 +92,12 @@ func GenerateSecp256r1Key() (*ecdsa.PrivateKey, error) { return ecdsa.GenerateKey(elliptic.P256(), rand.Reader) } -// EncodeSecp256k1PrivateKey encodes raw ECDSA private key. +// EncodeSecp256r1PrivateKey encodes raw ECDSA private key. func EncodeSecp256r1PrivateKey(k *ecdsa.PrivateKey) ([]byte, error) { return x509.MarshalECPrivateKey(k) } -// DecodeSecp256k1PrivateKey decodes raw ECDSA private key. +// DecodeSecp256r1PrivateKey decodes raw ECDSA private key. func DecodeSecp256r1PrivateKey(data []byte) (*ecdsa.PrivateKey, error) { return x509.ParseECPrivateKey(data) } diff --git a/pkg/file/pipeline/hashtrie/hashtrie.go b/pkg/file/pipeline/hashtrie/hashtrie.go index 7ef5eb350a1..1aff8c0df66 100644 --- a/pkg/file/pipeline/hashtrie/hashtrie.go +++ b/pkg/file/pipeline/hashtrie/hashtrie.go @@ -109,7 +109,7 @@ func (h *hashTrieWriter) writeToDataLevel(span, ref, key, data []byte) error { return h.rParams.ChunkWrite(0, data, h.parityChunkFn) } -// wrapLevel wraps an existing level and writes the resulting hash to the following level +// wrapFullLevel wraps an existing level and writes the resulting hash to the following level // then truncates the current level data by shifting the cursors. // Steps are performed in the following order: // - take all of the data in the current level diff --git a/pkg/p2p/libp2p/peer.go b/pkg/p2p/libp2p/peer.go index 5aa0f75713a..7966454f295 100644 --- a/pkg/p2p/libp2p/peer.go +++ b/pkg/p2p/libp2p/peer.go @@ -51,7 +51,7 @@ func (r *peerRegistry) Exists(overlay swarm.Address) (found bool) { return found } -// Disconnect removes the peer from registry in disconnect. +// Disconnected removes the peer from registry in disconnect. // peerRegistry has to be set by network.Network.Notify(). func (r *peerRegistry) Disconnected(_ network.Network, c network.Conn) { peerID := c.RemotePeer() From 6e07242bc930bbadc493c64c9bd631797631c537 Mon Sep 17 00:00:00 2001 From: zhedazijingang Date: Tue, 9 Sep 2025 15:35:52 +0800 Subject: [PATCH 16/96] refactor: using reflect.TypeFor (#5200) Signed-off-by: zhedazijingang --- pkg/api/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/api/util.go b/pkg/api/util.go index 6fdb575873a..340a6270b5f 100644 --- a/pkg/api/util.go +++ b/pkg/api/util.go @@ -253,7 +253,7 @@ func mapStructure(input, output interface{}, hooks map[string]func(v string) (st } case reflect.Interface: switch field.Type() { - case reflect.TypeOf((*multiaddr.Multiaddr)(nil)).Elem(): + case reflect.TypeFor[multiaddr.Multiaddr](): val, err := multiaddr.NewMultiaddr(value) if err != nil { return err From 059c771488403f87e46cbd2c20ef1097a08b1f6a Mon Sep 17 00:00:00 2001 From: Olexandr88 Date: Tue, 9 Sep 2025 15:03:21 +0300 Subject: [PATCH 17/96] fix(docs): update license badge link (#5207) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ad6caf49529..c67ffcce452 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ [![API OpenAPI Specs](https://img.shields.io/badge/openapi-api-blue)](https://docs.ethswarm.org/api/) ![Docker Pulls](https://img.shields.io/docker/pulls/ethersphere/bee) ![GitHub all releases](https://img.shields.io/github/downloads/ethersphere/bee/total) -![GitHub](https://img.shields.io/github/license/ethersphere/bee) +[![GitHub](https://img.shields.io/github/license/ethersphere/bee)](https://github.com/ethersphere/bee/blob/master/LICENSE) ## DISCLAIMER From 46f99ab23df19819a197c5a366e3a63872784d39 Mon Sep 17 00:00:00 2001 From: nugaon Date: Tue, 9 Sep 2025 15:48:02 +0200 Subject: [PATCH 18/96] fix: uncompressed eth message signature (#5197) --- pkg/crypto/signer.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/crypto/signer.go b/pkg/crypto/signer.go index 02863ed2d6e..9c3ee670ded 100644 --- a/pkg/crypto/signer.go +++ b/pkg/crypto/signer.go @@ -89,7 +89,7 @@ func (d *defaultSigner) Sign(data []byte) (signature []byte, err error) { return nil, err } - return d.sign(hash, true) + return d.sign(hash, false) } // SignTx signs an ethereum transaction. @@ -140,7 +140,7 @@ func (d *defaultSigner) SignTypedData(typedData *eip712.TypedData) ([]byte, erro // sign the provided hash and convert it to the ethereum (r,s,v) format. func (d *defaultSigner) sign(sighash []byte, isCompressedKey bool) ([]byte, error) { pvk, _ := btcec.PrivKeyFromBytes(d.key.D.Bytes()) - signature, err := btcecdsa.SignCompact(pvk, sighash, false) + signature, err := btcecdsa.SignCompact(pvk, sighash, isCompressedKey) if err != nil { return nil, err } From cb09d3e4937b8dd7a08330dcde1aa7ac1eb72f44 Mon Sep 17 00:00:00 2001 From: keeghcet Date: Wed, 10 Sep 2025 15:15:25 +0800 Subject: [PATCH 19/96] refactor: use b.Loop() to simplify the code (#5206) Signed-off-by: keeghcet --- pkg/bmt/benchmark_test.go | 20 ++++++++++---------- pkg/feeds/epochs/lookup_benchmark_test.go | 2 +- pkg/feeds/sequence/lookup_benchmark_test.go | 2 +- pkg/file/pipeline/builder/builder_test.go | 2 +- pkg/file/splitter/splitter_test.go | 2 +- pkg/pss/mining_test.go | 2 +- pkg/storage/storagetest/benchmark.go | 12 ++++++------ pkg/storage/storagetest/storage.go | 8 ++++---- pkg/topology/pslice/pslice_test.go | 20 +++++--------------- 9 files changed, 30 insertions(+), 40 deletions(-) diff --git a/pkg/bmt/benchmark_test.go b/pkg/bmt/benchmark_test.go index acf0ab4fc83..7ab2339c02d 100644 --- a/pkg/bmt/benchmark_test.go +++ b/pkg/bmt/benchmark_test.go @@ -47,8 +47,8 @@ func benchmarkSHA3(b *testing.B, n int) { testData := testutil.RandBytesWithSeed(b, 4096, seed) b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { + + for b.Loop() { if _, err := bmt.Sha3hash(testData[:n]); err != nil { b.Fatalf("seed %d: %v", seed, err) } @@ -66,8 +66,8 @@ func benchmarkBMTBaseline(b *testing.B, _ int) { testData := testutil.RandBytesWithSeed(b, 4096, seed) b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { + + for b.Loop() { eg := new(errgroup.Group) for j := 0; j < testSegmentCount; j++ { eg.Go(func() error { @@ -92,8 +92,8 @@ func benchmarkBMT(b *testing.B, n int) { defer pool.Put(h) b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { + + for b.Loop() { if _, err := syncHash(h, testData[:n]); err != nil { b.Fatalf("seed %d: %v", seed, err) } @@ -110,8 +110,8 @@ func benchmarkPool(b *testing.B, poolsize int) { cycles := 100 b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { + + for b.Loop() { eg := new(errgroup.Group) for j := 0; j < cycles; j++ { eg.Go(func() error { @@ -136,8 +136,8 @@ func benchmarkRefHasher(b *testing.B, n int) { rbmt := reference.NewRefHasher(swarm.NewHasher(), 128) b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { + + for b.Loop() { _, err := rbmt.Hash(testData[:n]) if err != nil { b.Fatal(err) diff --git a/pkg/feeds/epochs/lookup_benchmark_test.go b/pkg/feeds/epochs/lookup_benchmark_test.go index 8f964bc553b..9d1005ab023 100644 --- a/pkg/feeds/epochs/lookup_benchmark_test.go +++ b/pkg/feeds/epochs/lookup_benchmark_test.go @@ -58,7 +58,7 @@ func BenchmarkFinder(b *testing.B) { } { names := []string{"sync", "async"} b.Run(fmt.Sprintf("%s:prefill=%d, latest=%d, now=%d", names[k], prefill, latest, now), func(b *testing.B) { - for n := 0; n < b.N; n++ { + for b.Loop() { _, _, _, err := finder.At(ctx, int64(now), after) if err != nil { b.Fatal(err) diff --git a/pkg/feeds/sequence/lookup_benchmark_test.go b/pkg/feeds/sequence/lookup_benchmark_test.go index e81d8cc9a61..d10fe144aca 100644 --- a/pkg/feeds/sequence/lookup_benchmark_test.go +++ b/pkg/feeds/sequence/lookup_benchmark_test.go @@ -54,7 +54,7 @@ func BenchmarkFinder(b *testing.B) { } { names := []string{"sync", "async"} b.Run(fmt.Sprintf("%s:prefill=%d, latest/now=%d", names[k], prefill, now), func(b *testing.B) { - for n := 0; n < b.N; n++ { + for b.Loop() { _, _, _, err := finder.At(ctx, now, 0) if err != nil { b.Fatal(err) diff --git a/pkg/file/pipeline/builder/builder_test.go b/pkg/file/pipeline/builder/builder_test.go index 17091bcc0d7..5b5d4321c63 100644 --- a/pkg/file/pipeline/builder/builder_test.go +++ b/pkg/file/pipeline/builder/builder_test.go @@ -140,7 +140,7 @@ func BenchmarkPipeline(b *testing.B) { 100000000, // 100 mb } { b.Run(strconv.Itoa(count)+"-bytes", func(b *testing.B) { - for n := 0; n < b.N; n++ { + for b.Loop() { benchmarkPipeline(b, count) } }) diff --git a/pkg/file/splitter/splitter_test.go b/pkg/file/splitter/splitter_test.go index 04958f5dd82..c54991f9579 100644 --- a/pkg/file/splitter/splitter_test.go +++ b/pkg/file/splitter/splitter_test.go @@ -223,7 +223,7 @@ func BenchmarkSplitter(b *testing.B) { 100000000, // 100 mb } { b.Run(strconv.Itoa(count)+"-bytes", func(b *testing.B) { - for n := 0; n < b.N; n++ { + for b.Loop() { benchmarkSplitter(b, count) } }) diff --git a/pkg/pss/mining_test.go b/pkg/pss/mining_test.go index 4dd94e45bb2..1cb4cc03282 100644 --- a/pkg/pss/mining_test.go +++ b/pkg/pss/mining_test.go @@ -51,7 +51,7 @@ func BenchmarkWrap(b *testing.B) { name := fmt.Sprintf("length:%d,depth:%d", c.length, c.depth) b.Run(name, func(b *testing.B) { targets := newTargets(c.length, c.depth) - for i := 0; i < b.N; i++ { + for b.Loop() { if _, err := pss.Wrap(ctx, topic, msg, pubkey, targets); err != nil { b.Fatal(err) } diff --git a/pkg/storage/storagetest/benchmark.go b/pkg/storage/storagetest/benchmark.go index ea01d58121c..fe2fe3dec12 100644 --- a/pkg/storage/storagetest/benchmark.go +++ b/pkg/storage/storagetest/benchmark.go @@ -227,7 +227,7 @@ func maxInt(a int, b int) int { func doRead(b *testing.B, db storage.Store, g keyGenerator, allowNotFound bool) { b.Helper() - for i := 0; i < b.N; i++ { + for i := 0; b.Loop(); i++ { key := g.Key(i) item := &obj1{ Id: string(key), @@ -269,7 +269,7 @@ func doWrite(b *testing.B, db storage.Store, g entryGenerator) { b.Helper() w := newDBWriter(db) - for i := 0; i < b.N; i++ { + for i := 0; b.Loop(); i++ { if err := w.Put(g.Key(i), g.Value(i)); err != nil { b.Fatalf("write key '%s': %v", string(g.Key(i)), err) } @@ -280,7 +280,7 @@ func doDelete(b *testing.B, db storage.Store, g keyGenerator) { b.Helper() w := newDBWriter(db) - for i := 0; i < b.N; i++ { + for i := 0; b.Loop(); i++ { if err := w.Delete(g.Key(i)); err != nil { b.Fatalf("delete key '%s': %v", string(g.Key(i)), err) } @@ -304,7 +304,7 @@ func populate(b *testing.B, db storage.Store) { func doDeleteChunk(b *testing.B, db storage.ChunkStore, g keyGenerator) { b.Helper() - for i := 0; i < b.N; i++ { + for i := 0; b.Loop(); i++ { addr := swarm.MustParseHexAddress(string(g.Key(i))) if err := db.Delete(context.Background(), addr); err != nil { b.Fatalf("delete key '%s': %v", string(g.Key(i)), err) @@ -315,7 +315,7 @@ func doDeleteChunk(b *testing.B, db storage.ChunkStore, g keyGenerator) { func doWriteChunk(b *testing.B, db storage.Putter, g entryGenerator) { b.Helper() - for i := 0; i < b.N; i++ { + for i := 0; b.Loop(); i++ { buf := make([]byte, swarm.HashSize) if _, err := hex.Decode(buf, g.Key(i)); err != nil { b.Fatalf("decode value: %v", err) @@ -331,7 +331,7 @@ func doWriteChunk(b *testing.B, db storage.Putter, g entryGenerator) { func doReadChunk(b *testing.B, db storage.ChunkStore, g keyGenerator, allowNotFound bool) { b.Helper() - for i := 0; i < b.N; i++ { + for i := 0; b.Loop(); i++ { key := string(g.Key(i)) addr := swarm.MustParseHexAddress(key) _, err := db.Get(context.Background(), addr) diff --git a/pkg/storage/storagetest/storage.go b/pkg/storage/storagetest/storage.go index d385d3941fc..e40e6bb9f20 100644 --- a/pkg/storage/storagetest/storage.go +++ b/pkg/storage/storagetest/storage.go @@ -950,7 +950,7 @@ func BenchmarkWriteInBatches(b *testing.B, bs storage.BatchStore) { g := newSequentialEntryGenerator(b.N) batch := bs.Batch(context.Background()) resetBenchmark(b) - for i := 0; i < b.N; i++ { + for i := 0; b.Loop(); i++ { key := g.Key(i) item := &obj1{ Id: string(key), @@ -969,7 +969,7 @@ func BenchmarkWriteInFixedSizeBatches(b *testing.B, bs storage.BatchStore) { g := newSequentialEntryGenerator(b.N) writer := newBatchDBWriter(bs) resetBenchmark(b) - for i := 0; i < b.N; i++ { + for i := 0; b.Loop(); i++ { writer.Put(g.Key(i), g.Value(i)) } } @@ -1021,7 +1021,7 @@ func BenchmarkDeleteInBatches(b *testing.B, bs storage.BatchStore) { doWrite(b, bs, g) resetBenchmark(b) batch := bs.Batch(context.Background()) - for i := 0; i < b.N; i++ { + for i := 0; b.Loop(); i++ { item := &obj1{ Id: string(g.Key(i)), } @@ -1039,7 +1039,7 @@ func BenchmarkDeleteInFixedSizeBatches(b *testing.B, bs storage.BatchStore) { doWrite(b, bs, g) resetBenchmark(b) writer := newBatchDBWriter(bs) - for i := 0; i < b.N; i++ { + for i := 0; b.Loop(); i++ { writer.Delete(g.Key(i)) } } diff --git a/pkg/topology/pslice/pslice_test.go b/pkg/topology/pslice/pslice_test.go index ee1c26ebaaf..c7b540450b4 100644 --- a/pkg/topology/pslice/pslice_test.go +++ b/pkg/topology/pslice/pslice_test.go @@ -435,9 +435,7 @@ func BenchmarkAdd(b *testing.B) { addrs := swarm.RandAddresses(b, bins*perBin) - b.ResetTimer() - - for n := 0; n < b.N; n++ { + for b.Loop() { for _, addr := range addrs { ps.Add(addr) } @@ -450,9 +448,7 @@ func BenchmarkAddBatch(b *testing.B) { addrs := swarm.RandAddresses(b, bins*perBin) - b.ResetTimer() - - for n := 0; n < b.N; n++ { + for b.Loop() { ps.Add(addrs...) } } @@ -464,9 +460,7 @@ func BenchmarkRemove(b *testing.B) { addrs := swarm.RandAddresses(b, bins*perBin) ps.Add(addrs...) - b.ResetTimer() - - for n := 0; n < b.N; n++ { + for b.Loop() { for _, addr := range addrs { ps.Remove(addr) } @@ -480,9 +474,7 @@ func BenchmarkEachBin(b *testing.B) { addrs := swarm.RandAddresses(b, bins*perBin) ps.Add(addrs...) - b.ResetTimer() - - for n := 0; n < b.N; n++ { + for b.Loop() { _ = ps.EachBin(func(a swarm.Address, u uint8) (stop bool, jumpToNext bool, err error) { return false, false, nil }) @@ -496,9 +488,7 @@ func BenchmarkEachBinRev(b *testing.B) { addrs := swarm.RandAddresses(b, bins*perBin) ps.Add(addrs...) - b.ResetTimer() - - for n := 0; n < b.N; n++ { + for b.Loop() { _ = ps.EachBinRev(func(a swarm.Address, u uint8) (stop bool, jumpToNext bool, err error) { return false, false, nil }) From d16fd56d7cfc2b67be4ec3fd7b81f4ba02397fa7 Mon Sep 17 00:00:00 2001 From: Olexandr88 Date: Wed, 10 Sep 2025 17:00:40 +0300 Subject: [PATCH 20/96] fix(docs): update install instructions link (#5211) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c67ffcce452..34d71449e8c 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ Please read the [coding guidelines](CODING.md) and [style guide](CODINGSTYLE.md) ## Installing -[Install instructions](https://docs.ethswarm.org/docs/installation/quick-start) +[Install instructions](https://docs.ethswarm.org/docs/bee/installation/quick-start) ## Get in touch From a2487ff0996dbfc10c1dad345baf621cb40bce6d Mon Sep 17 00:00:00 2001 From: mickychang9 Date: Thu, 11 Sep 2025 22:37:13 +0800 Subject: [PATCH 21/96] chore: fix comment function name (#5212) Signed-off-by: mickychang9 --- pkg/api/chunk_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/api/chunk_test.go b/pkg/api/chunk_test.go index 35ad491e606..aa4e4ec485f 100644 --- a/pkg/api/chunk_test.go +++ b/pkg/api/chunk_test.go @@ -248,7 +248,7 @@ func TestChunkInvalidParams(t *testing.T) { }) } -// TestDirectChunkUpload tests that the direct upload endpoint give correct error message in dev mode +// TestChunkDirectUpload tests that the direct upload endpoint give correct error message in dev mode func TestChunkDirectUpload(t *testing.T) { t.Parallel() var ( From 364fbc5f46827dc2bc02f821bea095706d18081e Mon Sep 17 00:00:00 2001 From: jinjiadu Date: Thu, 11 Sep 2025 10:39:55 -0400 Subject: [PATCH 22/96] refactor: use the built-in max/min to simplify the code (#5103) Signed-off-by: jinjiadu --- pkg/bmt/reference/reference.go | 5 +---- pkg/manifest/mantaray/marshal.go | 10 ++-------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/pkg/bmt/reference/reference.go b/pkg/bmt/reference/reference.go index 8b399f8ffc0..416a7d4a354 100644 --- a/pkg/bmt/reference/reference.go +++ b/pkg/bmt/reference/reference.go @@ -38,10 +38,7 @@ func NewRefHasher(h hash.Hash, count int) *RefHasher { func (rh *RefHasher) Hash(data []byte) ([]byte, error) { // if data is shorter than the base length (maxDataLength), we provide padding with zeros d := make([]byte, rh.maxDataLength) - length := len(data) - if length > rh.maxDataLength { - length = rh.maxDataLength - } + length := min(len(data), rh.maxDataLength) copy(d, data[:length]) return rh.hash(d, rh.maxDataLength) } diff --git a/pkg/manifest/mantaray/marshal.go b/pkg/manifest/mantaray/marshal.go index 4e0324c35dd..0129312ff9f 100644 --- a/pkg/manifest/mantaray/marshal.go +++ b/pkg/manifest/mantaray/marshal.go @@ -162,10 +162,7 @@ func (n *Node) MarshalBinary() (bytes []byte, err error) { copy(xorEncryptedBytes, bytes[0:nodeObfuscationKeySize]) for i := nodeObfuscationKeySize; i < len(bytes); i += nodeObfuscationKeySize { - end := i + nodeObfuscationKeySize - if end > len(bytes) { - end = len(bytes) - } + end := min(i+nodeObfuscationKeySize, len(bytes)) encrypted := encryptDecrypt(bytes[i:end], n.obfuscationKey) copy(xorEncryptedBytes[i:end], encrypted) @@ -228,10 +225,7 @@ func (n *Node) UnmarshalBinary(data []byte) error { copy(xorDecryptedBytes, data[0:nodeObfuscationKeySize]) for i := nodeObfuscationKeySize; i < len(data); i += nodeObfuscationKeySize { - end := i + nodeObfuscationKeySize - if end > len(data) { - end = len(data) - } + end := min(i+nodeObfuscationKeySize, len(data)) decrypted := encryptDecrypt(data[i:end], n.obfuscationKey) copy(xorDecryptedBytes[i:end], decrypted) From 71c4fb97b280891b24bad5a08e1f32fdc677f338 Mon Sep 17 00:00:00 2001 From: nugaon Date: Fri, 12 Sep 2025 16:09:39 +0200 Subject: [PATCH 23/96] feat: feed legacy payload param (#5163) --- openapi/Swarm.yaml | 3 ++ openapi/SwarmCommon.yaml | 8 +++ pkg/api/bzz.go | 36 +++++++++++-- pkg/api/bzz_test.go | 106 +++++++++++++++++++++++++++------------ pkg/api/feed.go | 7 +-- pkg/api/feed_test.go | 47 +++++++++++++---- pkg/api/subdomain.go | 10 +++- pkg/feeds/getter.go | 31 +++++++----- pkg/feeds/getter_test.go | 37 ++++++++++---- pkg/node/bootstrap.go | 2 +- 10 files changed, 215 insertions(+), 72 deletions(-) diff --git a/openapi/Swarm.yaml b/openapi/Swarm.yaml index f3fa6189dc3..48916c54af1 100644 --- a/openapi/Swarm.yaml +++ b/openapi/Swarm.yaml @@ -435,6 +435,7 @@ paths: $ref: "SwarmCommon.yaml#/components/schemas/SwarmReference" required: true description: Swarm address of content + - $ref: "SwarmCommon.yaml#/components/parameters/SwarmFeedLegacyResolve" - $ref: "SwarmCommon.yaml#/components/parameters/SwarmCache" - $ref: "SwarmCommon.yaml#/components/parameters/SwarmRedundancyStrategyParameter" - $ref: "SwarmCommon.yaml#/components/parameters/SwarmRedundancyFallbackModeParameter" @@ -503,6 +504,7 @@ paths: type: string required: true description: Path to the file in the collection. + - $ref: "SwarmCommon.yaml#/components/parameters/SwarmFeedLegacyResolve" - $ref: "SwarmCommon.yaml#/components/parameters/SwarmRedundancyStrategyParameter" - $ref: "SwarmCommon.yaml#/components/parameters/SwarmRedundancyFallbackModeParameter" - $ref: "SwarmCommon.yaml#/components/parameters/SwarmChunkRetrievalTimeoutParameter" @@ -1036,6 +1038,7 @@ paths: $ref: "SwarmCommon.yaml#/components/schemas/FeedType" required: false description: "Feed indexing scheme (default: sequence)" + - $ref: "SwarmCommon.yaml#/components/parameters/SwarmFeedLegacyResolve" - $ref: "SwarmCommon.yaml#/components/parameters/SwarmOnlyRootChunkParameter" - $ref: "SwarmCommon.yaml#/components/parameters/SwarmCache" - $ref: "SwarmCommon.yaml#/components/parameters/SwarmRedundancyStrategyParameter" diff --git a/openapi/SwarmCommon.yaml b/openapi/SwarmCommon.yaml index 082b8494fdb..3d08175f01c 100644 --- a/openapi/SwarmCommon.yaml +++ b/openapi/SwarmCommon.yaml @@ -1282,6 +1282,14 @@ components: required: false description: "ACT history Unix timestamp" + SwarmFeedLegacyResolve: + in: query + name: swarm-feed-legacy-resolve + schema: + type: boolean + required: false + description: "Resolves feed payloads in legacy structure (timestamp, content address)." + responses: "200": description: OK. diff --git a/pkg/api/bzz.go b/pkg/api/bzz.go index e692091aa0d..a27e3f813c5 100644 --- a/pkg/api/bzz.go +++ b/pkg/api/bzz.go @@ -337,7 +337,15 @@ func (s *Service) bzzDownloadHandler(w http.ResponseWriter, r *http.Request) { paths.Path = strings.TrimRight(paths.Path, "/") + "/" // NOTE: leave one slash if there was some. } - s.serveReference(logger, address, paths.Path, w, r, false) + queries := struct { + FeedLegacyResolve bool `map:"swarm-feed-legacy-resolve"` + }{} + if response := s.mapStructure(r.URL.Query(), &queries); response != nil { + response("invalid query params", logger, w) + return + } + + s.serveReference(logger, address, paths.Path, w, r, false, queries.FeedLegacyResolve) } func (s *Service) bzzHeadHandler(w http.ResponseWriter, r *http.Request) { @@ -352,6 +360,14 @@ func (s *Service) bzzHeadHandler(w http.ResponseWriter, r *http.Request) { return } + queries := struct { + FeedLegacyResolve bool `map:"swarm-feed-legacy-resolve"` + }{} + if response := s.mapStructure(r.URL.Query(), &queries); response != nil { + response("invalid query params", logger, w) + return + } + address := paths.Address if v := getAddressFromContext(r.Context()); !v.IsZero() { address = v @@ -361,10 +377,10 @@ func (s *Service) bzzHeadHandler(w http.ResponseWriter, r *http.Request) { paths.Path = strings.TrimRight(paths.Path, "/") + "/" // NOTE: leave one slash if there was some. } - s.serveReference(logger, address, paths.Path, w, r, true) + s.serveReference(logger, address, paths.Path, w, r, true, queries.FeedLegacyResolve) } -func (s *Service) serveReference(logger log.Logger, address swarm.Address, pathVar string, w http.ResponseWriter, r *http.Request, headerOnly bool) { +func (s *Service) serveReference(logger log.Logger, address swarm.Address, pathVar string, w http.ResponseWriter, r *http.Request, headerOnly bool, feedLegacyResolve bool) { loggerV1 := logger.V(1).Build() headers := struct { @@ -433,8 +449,20 @@ FETCH: jsonhttp.NotFound(w, "no update found") return } - wc, err := feeds.GetWrappedChunk(ctx, s.storer.Download(cache), ch) + wc, err := feeds.GetWrappedChunk(ctx, s.storer.Download(cache), ch, feedLegacyResolve) if err != nil { + if errors.Is(err, feeds.ErrNotLegacyPayload) { + logger.Debug("bzz: download: feed is not a legacy payload") + logger.Error(err, "bzz download: feed is not a legacy payload") + jsonhttp.BadRequest(w, "bzz download: feed is not a legacy payload") + return + } + if errors.As(err, &feeds.WrappedChunkNotFoundError{}) { + logger.Debug("bzz download: feed pointing to the wrapped chunk not found", "error", err) + logger.Error(err, "bzz download: feed pointing to the wrapped chunk not found") + jsonhttp.NotFound(w, "bzz download: feed pointing to the wrapped chunk not found") + return + } logger.Debug("bzz download: mapStructure feed update failed", "error", err) logger.Error(nil, "bzz download: mapStructure feed update failed") jsonhttp.InternalServerError(w, "mapStructure feed update") diff --git a/pkg/api/bzz_test.go b/pkg/api/bzz_test.go index d69eb79df95..9738cb50f17 100644 --- a/pkg/api/bzz_test.go +++ b/pkg/api/bzz_test.go @@ -13,6 +13,7 @@ import ( "mime" "mime/multipart" "net/http" + "net/url" "strconv" "strings" "testing" @@ -26,6 +27,7 @@ import ( "github.com/ethersphere/bee/v2/pkg/manifest" mockbatchstore "github.com/ethersphere/bee/v2/pkg/postage/batchstore/mock" mockpost "github.com/ethersphere/bee/v2/pkg/postage/mock" + testingsoc "github.com/ethersphere/bee/v2/pkg/soc/testing" "github.com/ethersphere/bee/v2/pkg/storage/inmemchunkstore" mockstorer "github.com/ethersphere/bee/v2/pkg/storer/mock" "github.com/ethersphere/bee/v2/pkg/swarm" @@ -759,11 +761,24 @@ func TestFeedIndirection(t *testing.T) { updateData = []byte("

Swarm Feeds Hello World!

") logger = log.Noop storer = mockstorer.New() + ctx = context.Background() client, _, _, _ = newTestServer(t, testServerOptions{ Storer: storer, Logger: logger, Post: mockpost.New(mockpost.WithAcceptAll()), }) + bzzDownloadResource = func(addr, path string, legacyFeed bool) string { + values := url.Values{} + if legacyFeed { + values.Set("swarm-feed-legacy-resolve", strconv.FormatBool(legacyFeed)) + } + + baseURL := "/bzz/" + addr + "/" + path + if len(values) > 0 { + return baseURL + "?" + values.Encode() + } + return baseURL + } ) // tar all the test case files tarReader := tarFiles(t, []f{ @@ -794,29 +809,6 @@ func TestFeedIndirection(t *testing.T) { t.Fatalf("expected file reference, did not got any") } - // now use the "content" to mock the feed lookup - // also, use the mocked mantaray chunks that unmarshal - // into a real manifest with the mocked feed values when - // called from the bzz endpoint. then call the bzz endpoint with - // the pregenerated feed root manifest hash - - feedUpdate := toChunk(t, 121212, resp.Reference.Bytes()) - - var ( - look = newMockLookup(-1, 0, feedUpdate, nil, &id{}, nil) - factory = newMockFactory(look) - bzzDownloadResource = func(addr, path string) string { return "/bzz/" + addr + "/" + path } - ctx = context.Background() - ) - client, _, _, _ = newTestServer(t, testServerOptions{ - Storer: storer, - Logger: logger, - Feeds: factory, - }) - err := storer.Cache().Put(ctx, feedUpdate) - if err != nil { - t.Fatal(err) - } m, err := manifest.NewDefaultManifest( loadsave.New(storer.ChunkStore(), storer.Cache(), pipelineFactory(storer.Cache(), false, 0), redundancy.DefaultLevel), false, @@ -838,14 +830,66 @@ func TestFeedIndirection(t *testing.T) { t.Fatal(err) } - jsonhttptest.Request(t, client, http.MethodGet, bzzDownloadResource(manifRef.String(), ""), http.StatusOK, - jsonhttptest.WithExpectedResponse(updateData), - jsonhttptest.WithExpectedContentLength(len(updateData)), - jsonhttptest.WithExpectedResponseHeader(api.AccessControlExposeHeaders, api.SwarmFeedIndexHeader), - jsonhttptest.WithExpectedResponseHeader(api.AccessControlExposeHeaders, api.ContentDispositionHeader), - jsonhttptest.WithExpectedResponseHeader(api.ContentDispositionHeader, `inline; filename="index.html"`), - jsonhttptest.WithExpectedResponseHeader(api.ContentTypeHeader, "text/html; charset=utf-8"), - ) + // now use the "content" root chunk to mock the feed lookup + // also, use the mocked mantaray chunks that unmarshal + // into a real manifest with the mocked feed values when + // called from the bzz endpoint. then call the bzz endpoint with + // the pregenerated feed root manifest hash + + t.Run("legacy feed", func(t *testing.T) { + feedUpdate := toChunk(t, 121212, resp.Reference.Bytes()) + + var ( + look = newMockLookup(-1, 0, feedUpdate, nil, &id{}, nil) + factory = newMockFactory(look) + ) + client, _, _, _ = newTestServer(t, testServerOptions{ + Storer: storer, + Logger: logger, + Feeds: factory, + }) + + jsonhttptest.Request(t, client, http.MethodGet, bzzDownloadResource(manifRef.String(), "", true), http.StatusOK, + jsonhttptest.WithExpectedResponse(updateData), + jsonhttptest.WithExpectedContentLength(len(updateData)), + jsonhttptest.WithExpectedResponseHeader(api.AccessControlExposeHeaders, api.SwarmFeedIndexHeader), + jsonhttptest.WithExpectedResponseHeader(api.AccessControlExposeHeaders, api.ContentDispositionHeader), + jsonhttptest.WithExpectedResponseHeader(api.ContentDispositionHeader, `inline; filename="index.html"`), + jsonhttptest.WithExpectedResponseHeader(api.ContentTypeHeader, "text/html; charset=utf-8"), + ) + + jsonhttptest.Request(t, client, http.MethodGet, bzzDownloadResource(manifRef.String(), "", false), http.StatusNotFound) + }) + + t.Run("wrapped feed", func(t *testing.T) { + // get root chunk of data and wrap it in a feed + rootCh, err := storer.ChunkStore().Get(ctx, resp.Reference) + if err != nil { + t.Fatal(err) + } + socRootCh := testingsoc.GenerateMockSOC(t, rootCh.Data()[swarm.SpanSize:]).Chunk() + + var ( + look = newMockLookup(-1, 0, socRootCh, nil, &id{}, nil) + factory = newMockFactory(look) + ) + client, _, _, _ = newTestServer(t, testServerOptions{ + Storer: storer, + Logger: logger, + Feeds: factory, + }) + + jsonhttptest.Request(t, client, http.MethodGet, bzzDownloadResource(manifRef.String(), "", false), http.StatusOK, + jsonhttptest.WithExpectedResponse(updateData), + jsonhttptest.WithExpectedContentLength(len(updateData)), + jsonhttptest.WithExpectedResponseHeader(api.AccessControlExposeHeaders, api.SwarmFeedIndexHeader), + jsonhttptest.WithExpectedResponseHeader(api.AccessControlExposeHeaders, api.ContentDispositionHeader), + jsonhttptest.WithExpectedResponseHeader(api.ContentDispositionHeader, `inline; filename="index.html"`), + jsonhttptest.WithExpectedResponseHeader(api.ContentTypeHeader, "text/html; charset=utf-8"), + ) + + jsonhttptest.Request(t, client, http.MethodGet, bzzDownloadResource(manifRef.String(), "", true), http.StatusBadRequest) + }) } func Test_bzzDownloadHandler_invalidInputs(t *testing.T) { diff --git a/pkg/api/feed.go b/pkg/api/feed.go index a1345e2b39d..313b04c831a 100644 --- a/pkg/api/feed.go +++ b/pkg/api/feed.go @@ -53,8 +53,9 @@ func (s *Service) feedGetHandler(w http.ResponseWriter, r *http.Request) { } queries := struct { - At int64 `map:"at"` - After uint64 `map:"after"` + At int64 `map:"at"` + After uint64 `map:"after"` + FeedLegacyResolve bool `map:"swarm-feed-legacy-resolve"` }{} if response := s.mapStructure(r.URL.Query(), &queries); response != nil { response("invalid query params", logger, w) @@ -102,7 +103,7 @@ func (s *Service) feedGetHandler(w http.ResponseWriter, r *http.Request) { return } - wc, err := feeds.GetWrappedChunk(r.Context(), s.storer.Download(false), ch) + wc, err := feeds.GetWrappedChunk(r.Context(), s.storer.Download(false), ch, queries.FeedLegacyResolve) if err != nil { logger.Error(nil, "wrapped chunk cannot be retrieved") jsonhttp.NotFound(w, "wrapped chunk cannot be retrieved") diff --git a/pkg/api/feed_test.go b/pkg/api/feed_test.go index a30c7d086da..fcfe4b93e68 100644 --- a/pkg/api/feed_test.go +++ b/pkg/api/feed_test.go @@ -14,6 +14,8 @@ import ( "io" "math/big" "net/http" + "net/url" + "strconv" "testing" "github.com/ethersphere/bee/v2/pkg/api" @@ -42,11 +44,20 @@ func TestFeed_Get(t *testing.T) { t.Parallel() var ( - feedResource = func(owner, topic, at string) string { + feedResource = func(owner, topic, at string, legacyFeed bool) string { + values := url.Values{} if at != "" { - return fmt.Sprintf("/feeds/%s/%s?at=%s", owner, topic, at) + values.Set("at", at) } - return fmt.Sprintf("/feeds/%s/%s", owner, topic) + if legacyFeed { + values.Set("swarm-feed-legacy-resolve", strconv.FormatBool(legacyFeed)) + } + + baseURL := fmt.Sprintf("/feeds/%s/%s", owner, topic) + if len(values) > 0 { + return baseURL + "?" + values.Encode() + } + return baseURL } mockStorer = mockstorer.New() ) @@ -75,7 +86,7 @@ func TestFeed_Get(t *testing.T) { }) ) - jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", "12"), http.StatusOK, + jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", "12", true), http.StatusOK, jsonhttptest.WithExpectedResponse(mockWrappedCh.Data()[swarm.SpanSize:]), jsonhttptest.WithExpectedResponseHeader(api.SwarmFeedIndexHeader, hex.EncodeToString(idBytes)), jsonhttptest.WithExpectedResponseHeader(api.AccessControlExposeHeaders, api.SwarmFeedIndexHeader), @@ -86,7 +97,7 @@ func TestFeed_Get(t *testing.T) { ) }) - t.Run("latest", func(t *testing.T) { + t.Run("latest with legacy payload", func(t *testing.T) { t.Parallel() var ( @@ -102,7 +113,7 @@ func TestFeed_Get(t *testing.T) { }) ) - jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", ""), http.StatusOK, + jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", "", true), http.StatusOK, jsonhttptest.WithExpectedResponse(mockWrappedCh.Data()[swarm.SpanSize:]), jsonhttptest.WithExpectedContentLength(len(mockWrappedCh.Data()[swarm.SpanSize:])), jsonhttptest.WithExpectedResponseHeader(api.SwarmFeedIndexHeader, hex.EncodeToString(idBytes)), @@ -131,7 +142,7 @@ func TestFeed_Get(t *testing.T) { }) ) - jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", ""), http.StatusOK, + jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", "", false), http.StatusOK, jsonhttptest.WithExpectedResponse(testData), jsonhttptest.WithExpectedContentLength(len(testData)), jsonhttptest.WithExpectedResponseHeader(api.SwarmFeedIndexHeader, hex.EncodeToString(idBytes)), @@ -161,7 +172,23 @@ func TestFeed_Get(t *testing.T) { }) ) - jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", ""), http.StatusNotFound) + jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", "", true), http.StatusNotFound) + }) + + t.Run("query parameter legacy feed resolve", func(t *testing.T) { + t.Parallel() + + var ( + look = newMockLookup(1, 0, nil, errors.New("dummy"), &id{}, &id{}) + factory = newMockFactory(look) + client, _, _, _ = newTestServer(t, testServerOptions{ + Storer: mockStorer, + Feeds: factory, + }) + ) + + // Test with the legacyFeed parameter set to true which should add the query parameter + jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", "", false), http.StatusNotFound) }) t.Run("bigger payload than one chunk", func(t *testing.T) { @@ -193,7 +220,7 @@ func TestFeed_Get(t *testing.T) { ) t.Run("retrieve chunk tree", func(t *testing.T) { - jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", ""), http.StatusOK, + jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", "", false), http.StatusOK, jsonhttptest.WithExpectedResponse(testData), jsonhttptest.WithExpectedContentLength(testDataLen), jsonhttptest.WithExpectedResponseHeader(api.SwarmFeedIndexHeader, hex.EncodeToString(idBytes)), @@ -206,7 +233,7 @@ func TestFeed_Get(t *testing.T) { }) t.Run("retrieve only wrapped chunk", func(t *testing.T) { - jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", ""), http.StatusOK, + jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", "", false), http.StatusOK, jsonhttptest.WithRequestHeader(api.SwarmOnlyRootChunk, "true"), jsonhttptest.WithExpectedResponse(testRootCh.Data()), jsonhttptest.WithExpectedContentLength(len(testRootCh.Data())), diff --git a/pkg/api/subdomain.go b/pkg/api/subdomain.go index 5d5cede467a..b30e270a4ec 100644 --- a/pkg/api/subdomain.go +++ b/pkg/api/subdomain.go @@ -28,5 +28,13 @@ func (s *Service) subdomainHandler(w http.ResponseWriter, r *http.Request) { paths.Path = strings.TrimRight(paths.Path, "/") + "/" // NOTE: leave one slash if there was some. } - s.serveReference(logger, paths.Subdomain, paths.Path, w, r, false) + queries := struct { + FeedLegacyResolve bool `map:"swarm-feed-legacy-resolve"` + }{} + if response := s.mapStructure(r.URL.Query(), &queries); response != nil { + response("invalid query params", logger, w) + return + } + + s.serveReference(logger, paths.Subdomain, paths.Path, w, r, false, queries.FeedLegacyResolve) } diff --git a/pkg/feeds/getter.go b/pkg/feeds/getter.go index 5f902703ef8..d0ab55da024 100644 --- a/pkg/feeds/getter.go +++ b/pkg/feeds/getter.go @@ -15,7 +15,15 @@ import ( "github.com/ethersphere/bee/v2/pkg/swarm" ) -var errNotLegacyPayload = errors.New("feed update is not in the legacy payload structure") +var ErrNotLegacyPayload = errors.New("feed update is not in the legacy payload structure") + +type WrappedChunkNotFoundError struct { + Ref []byte +} + +func (e WrappedChunkNotFoundError) Error() string { + return fmt.Sprintf("feed pointing to the wrapped chunk not found: %x", e.Ref) +} // Lookup is the interface for time based feed lookup type Lookup interface { @@ -50,7 +58,7 @@ func (f *Getter) Get(ctx context.Context, i Index) (swarm.Chunk, error) { return f.getter.Get(ctx, addr) } -func GetWrappedChunk(ctx context.Context, getter storage.Getter, ch swarm.Chunk) (swarm.Chunk, error) { +func GetWrappedChunk(ctx context.Context, getter storage.Getter, ch swarm.Chunk, legacyResolve bool) (swarm.Chunk, error) { wc, err := FromChunk(ch) if err != nil { return nil, err @@ -59,16 +67,15 @@ func GetWrappedChunk(ctx context.Context, getter storage.Getter, ch swarm.Chunk) // possible values right now: // unencrypted ref: span+timestamp+ref => 8+8+32=48 // encrypted ref: span+timestamp+ref+decryptKey => 8+8+64=80 - ref, err := legacyPayload(wc) - if err != nil { - if errors.Is(err, errNotLegacyPayload) { - return wc, nil + if legacyResolve { + ref, err := legacyPayload(wc) + if err != nil { + return nil, err + } + wc, err = getter.Get(ctx, ref) + if err != nil { + return nil, WrappedChunkNotFoundError{Ref: ref.Bytes()} } - return nil, err - } - wc, err = getter.Get(ctx, ref) - if err != nil { - return nil, err } return wc, nil @@ -87,7 +94,7 @@ func FromChunk(ch swarm.Chunk) (swarm.Chunk, error) { func legacyPayload(wrappedChunk swarm.Chunk) (swarm.Address, error) { cacData := wrappedChunk.Data() if !(len(cacData) == 16+swarm.HashSize || len(cacData) == 16+swarm.HashSize*2) { - return swarm.ZeroAddress, errNotLegacyPayload + return swarm.ZeroAddress, ErrNotLegacyPayload } return swarm.NewAddress(cacData[16:]), nil diff --git a/pkg/feeds/getter_test.go b/pkg/feeds/getter_test.go index d09d5b837aa..df4f9b06c11 100644 --- a/pkg/feeds/getter_test.go +++ b/pkg/feeds/getter_test.go @@ -17,18 +17,24 @@ import ( func TestGetWrappedChunk(t *testing.T) { storer := mockstorer.New() + data := []byte("data") // new format (wraps chunk) - ch := soctesting.GenerateMockSOC(t, []byte("data")).Chunk() - wch, err := GetWrappedChunk(context.Background(), storer.ChunkStore(), ch) + ch := soctesting.GenerateMockSOC(t, data).Chunk() + wch, err := GetWrappedChunk(context.Background(), storer.ChunkStore(), ch, false) if err != nil { t.Fatal(err) } - if !bytes.Equal(wch.Data()[8:], []byte("data")) { + if !bytes.Equal(wch.Data()[8:], data) { t.Fatal("data mismatch") } // old format + err = storer.Put(context.Background(), wch) + if err != nil { + t.Fatal(err) + } + tt := []struct { name string addr []byte @@ -49,19 +55,30 @@ func TestGetWrappedChunk(t *testing.T) { binary.BigEndian.PutUint64(timestamp, 1) ch = soctesting.GenerateMockSOC(t, append(timestamp, tc.addr...)).Chunk() - err = storer.Put(context.Background(), wch) + wch, err = GetWrappedChunk(context.Background(), storer.ChunkStore(), ch, true) if err != nil { t.Fatal(err) } - wch, err = GetWrappedChunk(context.Background(), storer.ChunkStore(), ch) - if err != nil { - t.Fatal(err) - } - - if !bytes.Equal(wch.Data()[8:], []byte("data")) { + if !bytes.Equal(wch.Data()[8:], data) { t.Fatal("data mismatch") } }) } + + t.Run("returns feed legacy payload", func(t *testing.T) { + timestamp := make([]byte, 8) + binary.BigEndian.PutUint64(timestamp, 1) + feedChData := append(timestamp, wch.Address().Bytes()...) + ch = soctesting.GenerateMockSOC(t, feedChData).Chunk() + + wch, err = GetWrappedChunk(context.Background(), storer.ChunkStore(), ch, false) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(wch.Data()[8:], feedChData) { + t.Fatal("data should be similar as old legacy feed payload format.") + } + }) } diff --git a/pkg/node/bootstrap.go b/pkg/node/bootstrap.go index 2a6dc21661d..4ce709b16d9 100644 --- a/pkg/node/bootstrap.go +++ b/pkg/node/bootstrap.go @@ -336,7 +336,7 @@ func getLatestSnapshot( return nil, err } - return feeds.GetWrappedChunk(ctx, st, u) + return feeds.GetWrappedChunk(ctx, st, u, false) } func batchStoreExists(s storage.StateStorer) (bool, error) { From 0982c665537a68e5881346219af99bc416a1c278 Mon Sep 17 00:00:00 2001 From: riskrose Date: Fri, 12 Sep 2025 23:42:22 +0900 Subject: [PATCH 24/96] chore: remove duplicate package imports (#5079) Signed-off-by: riskrose Co-authored-by: Ljubisa Gacevic --- pkg/p2p/libp2p/libp2p.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/p2p/libp2p/libp2p.go b/pkg/p2p/libp2p/libp2p.go index 4d4c3773f92..8020b381708 100644 --- a/pkg/p2p/libp2p/libp2p.go +++ b/pkg/p2p/libp2p/libp2p.go @@ -55,7 +55,6 @@ import ( ocprom "contrib.go.opencensus.io/exporter/prometheus" m2 "github.com/ethersphere/bee/v2/pkg/metrics" - rcmgrObs "github.com/libp2p/go-libp2p/p2p/host/resource-manager" "github.com/prometheus/client_golang/prometheus" ) @@ -175,7 +174,7 @@ func New(ctx context.Context, signer beecrypto.Signer, networkID uint64, overlay } if o.Registry != nil { - rcmgrObs.MustRegisterWith(o.Registry) + rcmgr.MustRegisterWith(o.Registry) } _, err = ocprom.NewExporter(ocprom.Options{ @@ -201,7 +200,7 @@ func New(ctx context.Context, signer beecrypto.Signer, networkID uint64, overlay // The resource manager expects a limiter, se we create one from our limits. limiter := rcmgr.NewFixedLimiter(limits) - str, err := rcmgrObs.NewStatsTraceReporter() + str, err := rcmgr.NewStatsTraceReporter() if err != nil { return nil, err } From ca6e0407ac061bf31049f5313092f97a516da8f5 Mon Sep 17 00:00:00 2001 From: joemicky Date: Tue, 16 Sep 2025 17:49:26 +0800 Subject: [PATCH 25/96] refactor: use the built-in max/min to simplify the code (#5214) Signed-off-by: joemicky --- pkg/storer/cachestore.go | 7 +++---- pkg/storer/migration/refCntSize.go | 5 +---- pkg/storer/migration/reserveRepair.go | 5 +---- pkg/storer/reserve.go | 7 +++---- 4 files changed, 8 insertions(+), 16 deletions(-) diff --git a/pkg/storer/cachestore.go b/pkg/storer/cachestore.go index 6cc51e5550e..9a894a8683c 100644 --- a/pkg/storer/cachestore.go +++ b/pkg/storer/cachestore.go @@ -39,10 +39,9 @@ func (db *DB) cacheWorker(ctx context.Context) { continue } - evict := uint64(size - capc) - if evict < db.reserveOptions.cacheMinEvictCount { // evict at least a min count - evict = db.reserveOptions.cacheMinEvictCount - } + evict := max(uint64(size-capc), + // evict at least a min count + db.reserveOptions.cacheMinEvictCount) dur := captureDuration(time.Now()) err := db.cacheObj.RemoveOldest(ctx, db.storage, evict) diff --git a/pkg/storer/migration/refCntSize.go b/pkg/storer/migration/refCntSize.go index 64f6f58b315..2075c50effb 100644 --- a/pkg/storer/migration/refCntSize.go +++ b/pkg/storer/migration/refCntSize.go @@ -124,10 +124,7 @@ func RefCountSizeInc(s storage.BatchStore, logger log.Logger) func() error { } for i := 0; i < len(itemsToDelete); i += 10000 { - end := i + 10000 - if end > len(itemsToDelete) { - end = len(itemsToDelete) - } + end := min(i+10000, len(itemsToDelete)) b := s.Batch(context.Background()) for _, item := range itemsToDelete[i:end] { diff --git a/pkg/storer/migration/reserveRepair.go b/pkg/storer/migration/reserveRepair.go index ae6838fc1e7..938e6b04b28 100644 --- a/pkg/storer/migration/reserveRepair.go +++ b/pkg/storer/migration/reserveRepair.go @@ -117,10 +117,7 @@ func ReserveRepairer( batchSize := 1000 for i := 0; i < len(chunkBinItems); i += batchSize { - end := i + batchSize - if end > len(chunkBinItems) { - end = len(chunkBinItems) - } + end := min(i+batchSize, len(chunkBinItems)) err := st.Run(context.Background(), func(s transaction.Store) error { for _, item := range chunkBinItems[i:end] { err := s.IndexStore().Delete(item) diff --git a/pkg/storer/reserve.go b/pkg/storer/reserve.go index c79b2b9fb06..2ac9d318623 100644 --- a/pkg/storer/reserve.go +++ b/pkg/storer/reserve.go @@ -369,10 +369,9 @@ func (db *DB) unreserve(ctx context.Context) (err error) { default: } - evict := target - totalEvicted - if evict < int(db.reserveOptions.minEvictCount) { // evict at least a min count - evict = int(db.reserveOptions.minEvictCount) - } + evict := max(target-totalEvicted, + // evict at least a min count + int(db.reserveOptions.minEvictCount)) binEvicted, err := db.evictBatch(ctx, b, evict, radius) // eviction happens in batches, so we need to keep track of the total From 41f26a41bcdbf6e1b6ba5debe60b6fae9b6facdb Mon Sep 17 00:00:00 2001 From: zhoujiaweii Date: Fri, 19 Sep 2025 20:21:30 +0800 Subject: [PATCH 26/96] refactor: replace sort.Slice with slices.Sort for natural ordering (#5218) Signed-off-by: zhoujiaweii --- pkg/manifest/mantaray/walker.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/manifest/mantaray/walker.go b/pkg/manifest/mantaray/walker.go index a46533a72da..1a27ed23b8b 100644 --- a/pkg/manifest/mantaray/walker.go +++ b/pkg/manifest/mantaray/walker.go @@ -6,7 +6,7 @@ package mantaray import ( "context" - "sort" + "slices" ) // WalkNodeFunc is the type of the function called for each node visited @@ -34,7 +34,7 @@ func walkNode(ctx context.Context, path []byte, l Loader, n *Node, walkFn WalkNo for k := range n.forks { keys = append(keys, k) } - sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] }) + slices.Sort(keys) for _, k := range keys { v := n.forks[k] From 7bd8512cf88b135d4f3c43ed75921f8a29f68377 Mon Sep 17 00:00:00 2001 From: nugaon Date: Mon, 22 Sep 2025 09:39:31 +0200 Subject: [PATCH 27/96] fix: ens error handling (#5213) --- pkg/api/api.go | 29 ++++++++++++++++--- pkg/resolver/client/ens/ens.go | 10 +++++-- pkg/resolver/multiresolver/multiresolver.go | 6 ++++ .../multiresolver/multiresolver_test.go | 2 +- pkg/resolver/resolver.go | 2 +- 5 files changed, 40 insertions(+), 9 deletions(-) diff --git a/pkg/api/api.go b/pkg/api/api.go index e3419ccdb38..418744ad621 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -43,6 +43,7 @@ import ( "github.com/ethersphere/bee/v2/pkg/pss" "github.com/ethersphere/bee/v2/pkg/resolver" "github.com/ethersphere/bee/v2/pkg/resolver/client/ens" + "github.com/ethersphere/bee/v2/pkg/resolver/multiresolver" "github.com/ethersphere/bee/v2/pkg/sctx" "github.com/ethersphere/bee/v2/pkg/settlement" "github.com/ethersphere/bee/v2/pkg/settlement/swap" @@ -109,9 +110,8 @@ const ( ) const ( - multiPartFormData = "multipart/form-data" - contentTypeTar = "application/x-tar" - boolHeaderSetValue = "true" + multiPartFormData = "multipart/form-data" + contentTypeTar = "application/x-tar" ) var ( @@ -449,6 +449,10 @@ func (s *Service) resolveNameOrAddress(str string) (swarm.Address, error) { return addr, nil } + if errors.Is(err, multiresolver.ErrResolverService) || errors.Is(err, resolver.ErrServiceNotAvailable) { + return swarm.ZeroAddress, err + } + return swarm.ZeroAddress, fmt.Errorf("%w: %w", errInvalidNameOrAddress, err) } @@ -651,13 +655,23 @@ func (s *Service) mapStructure(input, output interface{}) func(string, log.Logge Message: msg, Code: http.StatusBadRequest, } + hasServiceUnavailable := false for _, err := range merr.Errors { + if errors.Is(err, resolver.ErrServiceNotAvailable) { + hasServiceUnavailable = true + resp.Reasons = append(resp.Reasons, jsonhttp.Reason{ + Field: "address", + Error: err.Error(), + }) + continue + } var perr *parseError if errors.As(err, &perr) { resp.Reasons = append(resp.Reasons, jsonhttp.Reason{ Field: perr.Entry, Error: perr.Cause.Error(), }) + continue } var verr *validationError if errors.As(err, &verr) { @@ -667,7 +681,14 @@ func (s *Service) mapStructure(input, output interface{}) func(string, log.Logge }) } } - jsonhttp.BadRequest(w, resp) + + if hasServiceUnavailable { + resp.Message = "service unavailable" + resp.Code = http.StatusServiceUnavailable + jsonhttp.ServiceUnavailable(w, resp) + } else { + jsonhttp.BadRequest(w, resp) + } } } diff --git a/pkg/resolver/client/ens/ens.go b/pkg/resolver/client/ens/ens.go index 38c7432e979..b921ad8c9b7 100644 --- a/pkg/resolver/client/ens/ens.go +++ b/pkg/resolver/client/ens/ens.go @@ -167,10 +167,10 @@ func wrapDial(endpoint, contractAddr string) (*ethclient.Client, *goens.Registry } func wrapResolve(registry *goens.Registry, _ common.Address, name string) (string, error) { - // Ensure the name is registered. ownerAddress, err := registry.Owner(name) + // it returns error only if the service is not available if err != nil { - return "", fmt.Errorf("owner: %w: %w", err, resolver.ErrNotFound) + return "", fmt.Errorf("%w: %w", resolver.ErrServiceNotAvailable, err) } // If the name is not registered, return an error. @@ -181,12 +181,16 @@ func wrapResolve(registry *goens.Registry, _ common.Address, name string) (strin // Obtain the resolver for this domain name. ensR, err := registry.Resolver(name) if err != nil { - return "", fmt.Errorf("resolver: %w: %w", err, resolver.ErrServiceNotAvailable) + return "", fmt.Errorf("%w: %w", resolver.ErrServiceNotAvailable, err) } // Try and read out the content hash record. ch, err := ensR.Contenthash() if err != nil { + // Check if it's a service error (rate limiting, network issues) + if strings.Contains(err.Error(), "429") || strings.Contains(err.Error(), "rate limit") { + return "", fmt.Errorf("%w: %w", resolver.ErrServiceNotAvailable, err) + } return "", fmt.Errorf("contenthash: %w: %w", err, resolver.ErrInvalidContentHash) } diff --git a/pkg/resolver/multiresolver/multiresolver.go b/pkg/resolver/multiresolver/multiresolver.go index e91f5155f28..87cfda56d16 100644 --- a/pkg/resolver/multiresolver/multiresolver.go +++ b/pkg/resolver/multiresolver/multiresolver.go @@ -14,6 +14,7 @@ import ( "github.com/ethersphere/bee/v2/pkg/resolver" "github.com/ethersphere/bee/v2/pkg/resolver/cidv1" "github.com/ethersphere/bee/v2/pkg/resolver/client/ens" + "github.com/ethersphere/bee/v2/pkg/swarm" "github.com/hashicorp/go-multierror" ) @@ -35,6 +36,8 @@ var ( ErrResolverChainFailed = errors.New("resolver chain failed") // ErrCloseFailed denotes that closing the multiresolver failed. ErrCloseFailed = errors.New("close failed") + // ErrResolverService denotes that no resolver service is configured for the requested name or the resolver service is not available. + ErrResolverService = errors.New("cannot communicate with the resolver or no resolver service configured") ) type resolverMap map[string][]resolver.Interface @@ -160,6 +163,9 @@ func (mr *MultiResolver) Resolve(name string) (addr resolver.Address, err error) // If no resolver chain is found, switch to the default chain. if len(chain) == 0 { chain = mr.resolvers[""] + if len(chain) == 1 && tld != "" { // only the CID resolver is defined + return swarm.ZeroAddress, resolver.ErrServiceNotAvailable + } } var errs *multierror.Error diff --git a/pkg/resolver/multiresolver/multiresolver_test.go b/pkg/resolver/multiresolver/multiresolver_test.go index c1f9ba0311a..8c242004696 100644 --- a/pkg/resolver/multiresolver/multiresolver_test.go +++ b/pkg/resolver/multiresolver/multiresolver_test.go @@ -217,7 +217,7 @@ func TestResolve(t *testing.T) { }, { // Switch to the default chain: - name: "this.empty", + name: "defaultChainIsCidTriggerItWithoutTld", wantAdr: addr, }, { diff --git a/pkg/resolver/resolver.go b/pkg/resolver/resolver.go index 043dcaf87e0..e289d454671 100644 --- a/pkg/resolver/resolver.go +++ b/pkg/resolver/resolver.go @@ -20,7 +20,7 @@ var ( // ErrNotFound denotes that given name was not found ErrNotFound = errors.New("not found") // ErrServiceNotAvailable denotes that remote ENS service is not available - ErrServiceNotAvailable = errors.New("not available") + ErrServiceNotAvailable = errors.New("ENS service is not available") // ErrInvalidContentHash denotes that the value of the response contenthash record is not valid. ErrInvalidContentHash = errors.New("invalid swarm content hash") ) From 7f997bde59b475fbb7b19a7c088c1f90b5b72fac Mon Sep 17 00:00:00 2001 From: nugaon Date: Tue, 23 Sep 2025 10:20:49 +0200 Subject: [PATCH 28/96] fix: add erasure reDecoder for evicted chunks (#5097) --- pkg/file/joiner/joiner.go | 52 ++++-- pkg/file/joiner/redecoder_test.go | 209 ++++++++++++++++++++++++ pkg/file/redundancy/getter/getter.go | 2 +- pkg/file/redundancy/getter/redecoder.go | 61 +++++++ 4 files changed, 309 insertions(+), 15 deletions(-) create mode 100644 pkg/file/joiner/redecoder_test.go create mode 100644 pkg/file/redundancy/getter/redecoder.go diff --git a/pkg/file/joiner/joiner.go b/pkg/file/joiner/joiner.go index 4d9152357c0..313ea21bd2c 100644 --- a/pkg/file/joiner/joiner.go +++ b/pkg/file/joiner/joiner.go @@ -65,9 +65,24 @@ func fingerprint(addrs []swarm.Address) string { return string(h.Sum(nil)) } +// createRemoveCallback returns a function that handles the cleanup after a recovery attempt +func (g *decoderCache) createRemoveCallback(key string) func(error) { + return func(err error) { + g.mu.Lock() + defer g.mu.Unlock() + if err != nil { + // signals that a new getter is needed to reattempt to recover the data + delete(g.cache, key) + } else { + // signals that the chunks were fetched/recovered/cached so a future getter is not needed + // The nil value indicates a successful recovery + g.cache[key] = nil + } + } +} + // GetOrCreate returns a decoder for the given chunk address func (g *decoderCache) GetOrCreate(addrs []swarm.Address, shardCnt int) storage.Getter { - // since a recovery decoder is not allowed, simply return the underlying netstore if g.config.Strict && g.config.Strategy == getter.NONE { return g.fetcher @@ -83,22 +98,31 @@ func (g *decoderCache) GetOrCreate(addrs []swarm.Address, shardCnt int) storage. d, ok := g.cache[key] if ok { if d == nil { - return g.fetcher + // The nil value indicates a previous successful recovery + // Create a new decoder but only use it as fallback if network fetch fails + decoderCallback := g.createRemoveCallback(key) + + // Create a factory function that will instantiate the decoder only when needed + recovery := func() storage.Getter { + g.config.Logger.Debug("lazy-creating recovery decoder after fetch failed", "key", key) + g.mu.Lock() + defer g.mu.Unlock() + d, ok := g.cache[key] + if ok && d != nil { + return d + } + d = getter.New(addrs, shardCnt, g.fetcher, g.putter, decoderCallback, g.config) + g.cache[key] = d + return d + } + + return getter.NewReDecoder(g.fetcher, recovery, g.config.Logger) } return d } - remove := func(err error) { - g.mu.Lock() - defer g.mu.Unlock() - if err != nil { - // signals that a new getter is needed to reattempt to recover the data - delete(g.cache, key) - } else { - // signals that the chunks were fetched/recovered/cached so a future getter is not needed - g.cache[key] = nil - } - } - d = getter.New(addrs, shardCnt, g.fetcher, g.putter, remove, g.config) + + removeCallback := g.createRemoveCallback(key) + d = getter.New(addrs, shardCnt, g.fetcher, g.putter, removeCallback, g.config) g.cache[key] = d return d } diff --git a/pkg/file/joiner/redecoder_test.go b/pkg/file/joiner/redecoder_test.go new file mode 100644 index 00000000000..7df041de76d --- /dev/null +++ b/pkg/file/joiner/redecoder_test.go @@ -0,0 +1,209 @@ +// Copyright 2025 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package joiner_test + +import ( + "bytes" + "context" + "strconv" + "testing" + + "github.com/ethersphere/bee/v2/pkg/cac" + "github.com/ethersphere/bee/v2/pkg/file/joiner" + "github.com/ethersphere/bee/v2/pkg/file/redundancy/getter" + "github.com/ethersphere/bee/v2/pkg/log" + "github.com/ethersphere/bee/v2/pkg/storage" + "github.com/ethersphere/bee/v2/pkg/storage/inmemchunkstore" + "github.com/ethersphere/bee/v2/pkg/swarm" + "github.com/klauspost/reedsolomon" +) + +// TestReDecoderFlow tests the complete flow of: +// 1. Loading data with redundancy getter +// 2. Successful recovery which nulls the decoder +// 3. Chunk eviction from cache +// 4. Reloading the same data through ReDecoder fallback +func TestReDecoderFlow(t *testing.T) { + ctx := context.Background() + dataShardCount := 4 + parityShardCount := 2 + totalShardCount := dataShardCount + parityShardCount + + // Create real data chunks with proper content + dataShards := make([][]byte, dataShardCount) + for i := 0; i < dataShardCount; i++ { + // Create chunks with simpler test data + dataShards[i] = make([]byte, swarm.ChunkWithSpanSize) + // Create a unique string for this shard + testData := []byte("test-data-" + strconv.Itoa(i)) + // Copy as much as will fit + copy(dataShards[i], testData) + } + + // Create parity chunks using Reed-Solomon encoding + parityShards := make([][]byte, parityShardCount) + for i := 0; i < parityShardCount; i++ { + parityShards[i] = make([]byte, swarm.ChunkWithSpanSize) + } + + // Create Reed-Solomon encoder + enc, err := reedsolomon.New(dataShardCount, parityShardCount) + if err != nil { + t.Fatalf("Failed to create Reed-Solomon encoder: %v", err) + } + + // Combine data and parity shards + allShards := make([][]byte, totalShardCount) + copy(allShards, dataShards) + copy(allShards[dataShardCount:], parityShards) + + // Encode to generate parity chunks + if err := enc.Encode(allShards); err != nil { + t.Fatalf("Failed to encode data: %v", err) + } + + // Create content-addressed chunks for all shards + addresses := make([]swarm.Address, totalShardCount) + chunks := make([]swarm.Chunk, totalShardCount) + + for i := 0; i < totalShardCount; i++ { + // Create proper content-addressed chunks + chunk, err := cac.NewWithDataSpan(allShards[i]) + if err != nil { + t.Fatalf("Failed to create content-addressed chunk %d: %v", i, err) + } + chunks[i] = chunk + addresses[i] = chunk.Address() + } + + // Select a data chunk to be missing (which will be recovered) + missingChunkIndex := 2 // The third data chunk will be missing + mockStore := inmemchunkstore.New() + netFetcher := newMockNetworkFetcher(addresses, addresses[missingChunkIndex]) + config := getter.Config{ + Strategy: getter.RACE, + Logger: log.Noop, + } + + j := joiner.NewDecoderCache(netFetcher, mockStore, config) + + // Step 1: Initializing decoder and triggering recovery + decoder := j.GetOrCreate(addresses, dataShardCount) + if decoder == nil { + t.Fatal("Failed to create decoder") + } + + // Verify we can now fetch the previously missing chunk through recovery + recoveredChunk, err := decoder.Get(ctx, addresses[missingChunkIndex]) + if err != nil { + t.Fatalf("Failed to recover missing chunk: %v", err) + } + // Verify the recovered chunk has the correct content + if !recoveredChunk.Address().Equal(addresses[missingChunkIndex]) { + t.Fatalf("Recovered chunk has incorrect address") + } + + // Verify the recovered chunk has the correct content + recoveredData := recoveredChunk.Data() + expectedData := chunks[missingChunkIndex].Data() + if len(recoveredData) != len(expectedData) { + t.Fatalf("Recovered chunk has incorrect data length: got %d, want %d", len(recoveredData), len(expectedData)) + } + if !bytes.Equal(recoveredData, expectedData) { + t.Fatalf("Recovered chunk has incorrect data") + } + // Check if the recovered chunk is now in the store + _, err = mockStore.Get(ctx, addresses[missingChunkIndex]) + if err != nil { + t.Fatalf("Recovered chunk not saved to store: %v", err) + } + + // Step 2: The original decoder should be automatically nulled after successful recovery + // This is an internal state check, we can't directly test it but we can verify that + // we can still access the chunks + + // Sanity check - verify we can still fetch chunks through the cache + for i := 0; i < dataShardCount; i++ { + _, err := decoder.Get(ctx, addresses[i]) + if err != nil { + t.Fatalf("Failed to get chunk %d after recovery: %v", i, err) + } + } + + // Step 3: Testing ReDecoder fallback + newDecoder := j.GetOrCreate(addresses, dataShardCount) + if newDecoder == nil { + t.Fatal("Failed to create ReDecoder") + } + + // Verify all chunks can be fetched through the ReDecoder + for i := 0; i < dataShardCount; i++ { + _, err := newDecoder.Get(ctx, addresses[i]) + if err != nil { + t.Fatalf("Failed to get chunk %d through ReDecoder: %v", i, err) + } + } + + // Verify that we can also access the first missing chunk - now from the store + // This would be using the local store and not network or recovery mechanisms + retrievedChunk, err := newDecoder.Get(ctx, addresses[missingChunkIndex]) + if err != nil { + t.Fatalf("Failed to retrieve previously recovered chunk: %v", err) + } + + if !retrievedChunk.Address().Equal(addresses[missingChunkIndex]) { + t.Fatalf("Retrieved chunk has incorrect address") + } + + // Also verify the data content matches + retrievedData := retrievedChunk.Data() + expectedData = chunks[missingChunkIndex].Data() + if len(retrievedData) != len(expectedData) { + t.Fatalf("Retrieved chunk has incorrect data length: got %d, want %d", len(retrievedData), len(expectedData)) + } + if !bytes.Equal(retrievedData, expectedData) { + t.Fatalf("Retrieved chunk has incorrect data") + } +} + +// Mock implementation of storage.Getter for testing +type mockNetworkFetcher struct { + allAddresses []swarm.Address + missingAddr swarm.Address +} + +// newMockNetworkFetcher creates a new mock fetcher that will return ErrNotFound for specific addresses +func newMockNetworkFetcher(allAddrs []swarm.Address, missingAddr swarm.Address) *mockNetworkFetcher { + return &mockNetworkFetcher{ + allAddresses: allAddrs, + missingAddr: missingAddr, + } +} + +// Get implements storage.Getter interface +func (m *mockNetworkFetcher) Get(ctx context.Context, addr swarm.Address) (swarm.Chunk, error) { + // Simulate network fetch - fail for the missing chunk + if addr.Equal(m.missingAddr) { + return nil, storage.ErrNotFound + } + + // Find the index of this address in the all addresses list + var index int + for i, a := range m.allAddresses { + if addr.Equal(a) { + index = i + break + } + } + + // Generate data using the same pattern as in the test + data := make([]byte, swarm.ChunkWithSpanSize) + // Create a unique string for this shard + testData := []byte("test-data-" + strconv.Itoa(index)) + // Copy as much as will fit + copy(data, testData) + + return swarm.NewChunk(addr, data), nil +} diff --git a/pkg/file/redundancy/getter/getter.go b/pkg/file/redundancy/getter/getter.go index 39bf23f87c3..0953c501dc5 100644 --- a/pkg/file/redundancy/getter/getter.go +++ b/pkg/file/redundancy/getter/getter.go @@ -340,7 +340,7 @@ func (g *decoder) setData(i int, chdata []byte) { func (g *decoder) getData(i int) []byte { g.mu.Lock() defer g.mu.Unlock() - if i == g.shardCnt-1 && g.lastLen > 0 { + if i == g.shardCnt-1 && g.lastLen > 0 && g.rsbuf[i] != nil { return g.rsbuf[i][:g.lastLen] // cut padding } return g.rsbuf[i] diff --git a/pkg/file/redundancy/getter/redecoder.go b/pkg/file/redundancy/getter/redecoder.go new file mode 100644 index 00000000000..0338a069841 --- /dev/null +++ b/pkg/file/redundancy/getter/redecoder.go @@ -0,0 +1,61 @@ +// Copyright 2025 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package getter + +import ( + "context" + "errors" + + "github.com/ethersphere/bee/v2/pkg/log" + "github.com/ethersphere/bee/v2/pkg/storage" + "github.com/ethersphere/bee/v2/pkg/swarm" +) + +// Recovery is a function that creates a recovery decoder on demand +type Recovery func() storage.Getter + +// ReDecoder is a wrapper around a Getter that first attempts to fetch a chunk directly +// from the network, and only falls back to recovery if the network fetch fails. +// This is used to handle cases where previously recovered chunks have been evicted from cache. +type ReDecoder struct { + fetcher storage.Getter // Direct fetcher (usually netstore) + recovery Recovery // Factory function to create recovery decoder on demand + logger log.Logger +} + +// NewReDecoder creates a new ReDecoder instance with the provided fetcher and recovery factory. +// The recovery decoder will only be created if needed (when network fetch fails). +func NewReDecoder(fetcher storage.Getter, recovery Recovery, logger log.Logger) *ReDecoder { + return &ReDecoder{ + fetcher: fetcher, + recovery: recovery, + logger: logger, + } +} + +// Get implements the storage.Getter interface. +// It first attempts to fetch the chunk directly from the network. +// If that fails with ErrNotFound, it then creates the recovery decoder and attempts to recover the chunk. +func (rd *ReDecoder) Get(ctx context.Context, addr swarm.Address) (swarm.Chunk, error) { + // First try to get the chunk directly from the network + chunk, err := rd.fetcher.Get(ctx, addr) + if err == nil { + return chunk, nil + } + + // Only attempt recovery if the chunk was not found + if !errors.Is(err, storage.ErrNotFound) { + return nil, err + } + + // Log that we're falling back to recovery + rd.logger.Debug("chunk not found in network, creating recovery decoder", "address", addr) + + // Create the recovery decoder on demand + recovery := rd.recovery() + + // Attempt to recover the chunk + return recovery.Get(ctx, addr) +} From 8e62840a91ca9d9f35a289fbe4ce98ba475cad11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ljubi=C5=A1a=20Ga=C4=8Devi=C4=87?= <35105035+gacevicljubisa@users.noreply.github.com> Date: Wed, 1 Oct 2025 14:12:59 +0200 Subject: [PATCH 29/96] feat(storer): add comprehensive metrics for ReserveSample performance (#5177) --- pkg/api/chequebook.go | 1 - pkg/api/pss.go | 5 ++-- pkg/storer/metrics.go | 61 +++++++++++++++++++++++++++++++------------ pkg/storer/sample.go | 50 +++++++++++++++++++++++++++++------ 4 files changed, 89 insertions(+), 28 deletions(-) diff --git a/pkg/api/chequebook.go b/pkg/api/chequebook.go index 33cd75d9767..219f9479089 100644 --- a/pkg/api/chequebook.go +++ b/pkg/api/chequebook.go @@ -22,7 +22,6 @@ import ( const ( errChequebookBalance = "cannot get chequebook balance" - errChequebookNoAmount = "did not specify amount" errChequebookNoWithdraw = "cannot withdraw" errChequebookNoDeposit = "cannot deposit" errChequebookInsufficientFunds = "insufficient funds" diff --git a/pkg/api/pss.go b/pkg/api/pss.go index ef1c3a84d47..03e9182585d 100644 --- a/pkg/api/pss.go +++ b/pkg/api/pss.go @@ -24,8 +24,7 @@ import ( ) const ( - writeDeadline = 4 * time.Second // write deadline. should be smaller than the shutdown timeout on api close - targetMaxLength = 3 // max target length in bytes, in order to prevent grieving by excess computation + writeDeadline = 4 * time.Second // write deadline. should be smaller than the shutdown timeout on api close ) func (s *Service) pssPostHandler(w http.ResponseWriter, r *http.Request) { @@ -42,7 +41,7 @@ func (s *Service) pssPostHandler(w http.ResponseWriter, r *http.Request) { topic := pss.NewTopic(paths.Topic) var targets pss.Targets - for _, v := range strings.Split(paths.Targets, ",") { + for v := range strings.SplitSeq(paths.Targets, ",") { target := struct { Val []byte `map:"target" validate:"required,max=3"` }{} diff --git a/pkg/storer/metrics.go b/pkg/storer/metrics.go index 7b9295ee9f7..68be82ac039 100644 --- a/pkg/storer/metrics.go +++ b/pkg/storer/metrics.go @@ -17,22 +17,24 @@ import ( // metrics groups storer related prometheus counters. type metrics struct { - MethodCalls *prometheus.CounterVec - MethodCallsDuration *prometheus.HistogramVec - ReserveSize prometheus.Gauge - ReserveSizeWithinRadius prometheus.Gauge - ReserveCleanup prometheus.Counter - StorageRadius prometheus.Gauge - CacheSize prometheus.Gauge - EvictedChunkCount prometheus.Counter - ExpiredChunkCount prometheus.Counter - OverCapTriggerCount prometheus.Counter - ExpiredBatchCount prometheus.Counter - LevelDBStats *prometheus.HistogramVec - ExpiryTriggersCount prometheus.Counter - ExpiryRunsCount prometheus.Counter - - ReserveMissingBatch prometheus.Gauge + MethodCalls *prometheus.CounterVec + MethodCallsDuration *prometheus.HistogramVec + ReserveSize prometheus.Gauge + ReserveSizeWithinRadius prometheus.Gauge + ReserveCleanup prometheus.Counter + StorageRadius prometheus.Gauge + CacheSize prometheus.Gauge + EvictedChunkCount prometheus.Counter + ExpiredChunkCount prometheus.Counter + OverCapTriggerCount prometheus.Counter + ExpiredBatchCount prometheus.Counter + LevelDBStats *prometheus.HistogramVec + ExpiryTriggersCount prometheus.Counter + ExpiryRunsCount prometheus.Counter + ReserveMissingBatch prometheus.Gauge + ReserveSampleDuration *prometheus.HistogramVec + ReserveSampleRunSummary *prometheus.GaugeVec + ReserveSampleLastRunTimestamp prometheus.Gauge } // newMetrics is a convenient constructor for creating new metrics. @@ -163,6 +165,33 @@ func newMetrics() metrics { Help: "Number of times the expiry worker was fired.", }, ), + ReserveSampleDuration: prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: m.Namespace, + Subsystem: subsystem, + Name: "reserve_sample_duration_seconds", + Help: "Duration of ReserveSample operations in seconds.", + Buckets: []float64{180, 300, 600, 900, 1200, 1500, 1800}, + }, + []string{"status"}, + ), + ReserveSampleRunSummary: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: m.Namespace, + Subsystem: subsystem, + Name: "reserve_sample_run_summary", + Help: "Summary metrics for the last ReserveSample run.", + }, + []string{"metric"}, + ), + ReserveSampleLastRunTimestamp: prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: m.Namespace, + Subsystem: subsystem, + Name: "reserve_sample_last_run_timestamp", + Help: "Unix timestamp of the last ReserveSample run completion.", + }, + ), } } diff --git a/pkg/storer/sample.go b/pkg/storer/sample.go index c2b17caecab..43999429300 100644 --- a/pkg/storer/sample.go +++ b/pkg/storer/sample.go @@ -75,8 +75,15 @@ func (db *DB) ReserveSample( statsLock.Unlock() } + workers := max(4, runtime.NumCPU()) t := time.Now() + defer func() { + duration := time.Since(t) + err := g.Wait() + db.recordReserveSampleMetrics(duration, allStats, workers, err) + }() + excludedBatchIDs, err := db.batchesBelowValue(minBatchBalance) if err != nil { db.logger.Error(err, "get batches below value") @@ -84,7 +91,6 @@ func (db *DB) ReserveSample( allStats.BatchesBelowValueDuration = time.Since(t) - workers := max(4, runtime.NumCPU()) chunkC := make(chan *reserve.ChunkBinItem, 3*workers) // Phase 1: Iterate chunk addresses @@ -133,7 +139,6 @@ func (db *DB) ReserveSample( // exclude chunks who's batches balance are below minimum if _, found := excludedBatchIDs[string(chItem.BatchID)]; found { wstat.BelowBalanceIgnored++ - continue } @@ -147,13 +152,15 @@ func (db *DB) ReserveSample( chunkLoadStart := time.Now() chunk, err := db.ChunkStore().Get(ctx, chItem.Address) + chunkLoadDuration := time.Since(chunkLoadStart) + if err != nil { wstat.ChunkLoadFailed++ db.logger.Debug("failed loading chunk", "chunk_address", chItem.Address, "error", err) continue } - wstat.ChunkLoadDuration += time.Since(chunkLoadStart) + wstat.ChunkLoadDuration += chunkLoadDuration taddrStart := time.Now() taddr, err := transformedAddress(hasher, chunk, chItem.ChunkType) @@ -224,8 +231,6 @@ func (db *DB) ReserveSample( } if le(item.TransformedAddress, currentMaxAddr) || len(sampleItems) < SampleSize { - start := time.Now() - stamp, err := chunkstamp.LoadWithBatchID(db.storage.IndexStore(), "reserve", item.ChunkAddress, item.Stamp.BatchID()) if err != nil { stats.StampLoadFailed++ @@ -241,13 +246,15 @@ func (db *DB) ReserveSample( continue } + stampValidStart := time.Now() if _, err := db.validStamp(ch); err != nil { stats.InvalidStamp++ db.logger.Debug("invalid stamp for chunk", "chunk_address", ch.Address(), "error", err) continue } - stats.ValidStampDuration += time.Since(start) + stampValidDuration := time.Since(stampValidStart) + stats.ValidStampDuration += stampValidDuration item.Stamp = postage.NewStamp(stamp.BatchID(), stamp.Index(), stamp.Timestamp(), stamp.Sig()) @@ -362,7 +369,6 @@ type SampleStats struct { func (s *SampleStats) add(other SampleStats) { s.TotalDuration += other.TotalDuration - s.TotalIterated += other.TotalIterated s.IterationDuration += other.IterationDuration s.SampleInserts += other.SampleInserts s.NewIgnored += other.NewIgnored @@ -375,6 +381,7 @@ func (s *SampleStats) add(other SampleStats) { s.ChunkLoadDuration += other.ChunkLoadDuration s.ChunkLoadFailed += other.ChunkLoadFailed s.StampLoadFailed += other.StampLoadFailed + s.TotalIterated += other.TotalIterated } // RandSample returns Sample with random values. @@ -382,7 +389,7 @@ func RandSample(t *testing.T, anchor []byte) Sample { t.Helper() chunks := make([]swarm.Chunk, SampleSize) - for i := 0; i < SampleSize; i++ { + for i := range SampleSize { ch := chunk.GenerateTestRandomChunk() if i%3 == 0 { ch = chunk.GenerateTestRandomSoChunk(t, ch) @@ -437,3 +444,30 @@ func getChunkType(chunk swarm.Chunk) swarm.ChunkType { } return swarm.ChunkTypeUnspecified } + +func (db *DB) recordReserveSampleMetrics(duration time.Duration, stats *SampleStats, workers int, err error) { + status := "success" + if err != nil { + status = "failure" + } + db.metrics.ReserveSampleDuration.WithLabelValues(status).Observe(duration.Seconds()) + + summaryMetrics := map[string]float64{ + "duration_seconds": duration.Seconds(), + "chunks_iterated": float64(stats.TotalIterated), + "chunks_load_failed": float64(stats.ChunkLoadFailed), + "stamp_validations": float64(stats.SampleInserts), + "invalid_stamps": float64(stats.InvalidStamp), + "below_balance_ignored": float64(stats.BelowBalanceIgnored), + "workers": float64(workers), + "chunks_per_second": float64(stats.TotalIterated) / duration.Seconds(), + "stamp_validation_duration_seconds": stats.ValidStampDuration.Seconds(), + "batches_below_value_duration_seconds": stats.BatchesBelowValueDuration.Seconds(), + } + + for metric, value := range summaryMetrics { + db.metrics.ReserveSampleRunSummary.WithLabelValues(metric).Set(value) + } + + db.metrics.ReserveSampleLastRunTimestamp.Set(float64(time.Now().Unix())) +} From 0856b6a5784a51587832b3c14568eb646d422d05 Mon Sep 17 00:00:00 2001 From: nugaon Date: Fri, 3 Oct 2025 16:02:43 +0200 Subject: [PATCH 30/96] fix: pin eviction guard (#5222) --- pkg/storer/internal/pinning/pinning.go | 26 ++++ pkg/storer/internal/reserve/reserve.go | 73 ++++++++++- pkg/storer/internal/reserve/reserve_test.go | 134 ++++++++++++++++++++ 3 files changed, 229 insertions(+), 4 deletions(-) diff --git a/pkg/storer/internal/pinning/pinning.go b/pkg/storer/internal/pinning/pinning.go index 04689a7eec6..01abe264fc8 100644 --- a/pkg/storer/internal/pinning/pinning.go +++ b/pkg/storer/internal/pinning/pinning.go @@ -207,6 +207,32 @@ func HasPin(st storage.Reader, root swarm.Address) (bool, error) { return has, nil } +// GetCollectionUUIDs returns all collection UUIDs from pin collections. +func GetCollectionUUIDs(st storage.Reader) ([][]byte, error) { + var collectionUUIDs [][]byte + err := st.Iterate(storage.Query{ + Factory: func() storage.Item { return &pinCollectionItem{} }, + }, func(r storage.Result) (bool, error) { + collection := r.Entry.(*pinCollectionItem) + collectionUUIDs = append(collectionUUIDs, collection.UUID) + return false, nil + }) + if err != nil { + return nil, fmt.Errorf("pin store: failed getting collections: %w", err) + } + return collectionUUIDs, nil +} + +// IsChunkPinnedInCollection checks if a chunk address is pinned under the given collection uuid. +func IsChunkPinnedInCollection(st storage.Reader, chunkAddr swarm.Address, uuid []byte) (bool, error) { + chunkItem := &pinChunkItem{UUID: uuid, Addr: chunkAddr} + has, err := st.Has(chunkItem) + if err != nil { + return false, fmt.Errorf("pin store: failed checking chunk pin status: %w", err) + } + return has, nil +} + // Pins lists all the added pinning collections. func Pins(st storage.Reader) ([]swarm.Address, error) { pins := make([]swarm.Address, 0) diff --git a/pkg/storer/internal/reserve/reserve.go b/pkg/storer/internal/reserve/reserve.go index 80e301e72a8..67a0043ccf5 100644 --- a/pkg/storer/internal/reserve/reserve.go +++ b/pkg/storer/internal/reserve/reserve.go @@ -19,6 +19,7 @@ import ( "github.com/ethersphere/bee/v2/pkg/postage" "github.com/ethersphere/bee/v2/pkg/storage" "github.com/ethersphere/bee/v2/pkg/storer/internal/chunkstamp" + pinstore "github.com/ethersphere/bee/v2/pkg/storer/internal/pinning" "github.com/ethersphere/bee/v2/pkg/storer/internal/stampindex" "github.com/ethersphere/bee/v2/pkg/storer/internal/transaction" "github.com/ethersphere/bee/v2/pkg/swarm" @@ -327,6 +328,7 @@ func (r *Reserve) Get(ctx context.Context, addr swarm.Address, batchID []byte, s } // EvictBatchBin evicts all chunks from bins upto the bin provided. +// Pinned chunks are protected from eviction to maintain data integrity. func (r *Reserve) EvictBatchBin( ctx context.Context, batchID []byte, @@ -336,13 +338,21 @@ func (r *Reserve) EvictBatchBin( r.multx.Lock(string(batchID)) defer r.multx.Unlock(string(batchID)) - var evicteditems []*BatchRadiusItem + var ( + evictedItems []*BatchRadiusItem + pinnedEvictedItems []*BatchRadiusItem + ) if count <= 0 { return 0, nil } - err := r.st.IndexStore().Iterate(storage.Query{ + pinUuids, err := pinstore.GetCollectionUUIDs(r.st.IndexStore()) + if err != nil { + return 0, err + } + + err = r.st.IndexStore().Iterate(storage.Query{ Factory: func() storage.Item { return &BatchRadiusItem{} }, Prefix: string(batchID), }, func(res storage.Result) (bool, error) { @@ -350,7 +360,24 @@ func (r *Reserve) EvictBatchBin( if batchRadius.Bin >= bin { return true, nil } - evicteditems = append(evicteditems, batchRadius) + + // Check if the chunk is pinned in any collection + pinned := false + for _, uuid := range pinUuids { + has, err := pinstore.IsChunkPinnedInCollection(r.st.IndexStore(), batchRadius.Address, uuid) + if err != nil { + return true, err + } + if has { + pinned = true + pinnedEvictedItems = append(pinnedEvictedItems, batchRadius) + break + } + } + + if !pinned { + evictedItems = append(evictedItems, batchRadius) + } count-- if count == 0 { return true, nil @@ -366,7 +393,7 @@ func (r *Reserve) EvictBatchBin( var evicted atomic.Int64 - for _, item := range evicteditems { + for _, item := range evictedItems { func(item *BatchRadiusItem) { eg.Go(func() error { err := r.st.Run(ctx, func(s transaction.Store) error { @@ -381,6 +408,21 @@ func (r *Reserve) EvictBatchBin( }(item) } + for _, item := range pinnedEvictedItems { + func(item *BatchRadiusItem) { + eg.Go(func() error { + err := r.st.Run(ctx, func(s transaction.Store) error { + return RemoveChunkMetaData(ctx, s, item) + }) + if err != nil { + return err + } + evicted.Add(1) + return nil + }) + }(item) + } + err = eg.Wait() r.size.Add(-evicted.Load()) @@ -430,6 +472,29 @@ func RemoveChunkWithItem( ) } +// RemoveChunkMetaData removes chunk reserve metadata from reserve indexes but keeps the cunks in the chunkstore. +// used at pinned data eviction +func RemoveChunkMetaData( + ctx context.Context, + trx transaction.Store, + item *BatchRadiusItem, +) error { + var errs error + + stamp, _ := chunkstamp.LoadWithStampHash(trx.IndexStore(), reserveScope, item.Address, item.StampHash) + if stamp != nil { + errs = errors.Join( + stampindex.Delete(trx.IndexStore(), reserveScope, stamp), + chunkstamp.DeleteWithStamp(trx.IndexStore(), reserveScope, item.Address, stamp), + ) + } + + return errors.Join(errs, + trx.IndexStore().Delete(item), + trx.IndexStore().Delete(&ChunkBinItem{Bin: item.Bin, BinID: item.BinID}), + ) +} + func (r *Reserve) IterateBin(bin uint8, startBinID uint64, cb func(swarm.Address, uint64, []byte, []byte) (bool, error)) error { err := r.st.IndexStore().Iterate(storage.Query{ Factory: func() storage.Item { return &ChunkBinItem{} }, diff --git a/pkg/storer/internal/reserve/reserve_test.go b/pkg/storer/internal/reserve/reserve_test.go index 95218e992f6..c6d682a7171 100644 --- a/pkg/storer/internal/reserve/reserve_test.go +++ b/pkg/storer/internal/reserve/reserve_test.go @@ -22,6 +22,7 @@ import ( chunk "github.com/ethersphere/bee/v2/pkg/storage/testing" "github.com/ethersphere/bee/v2/pkg/storer/internal" "github.com/ethersphere/bee/v2/pkg/storer/internal/chunkstamp" + pinstore "github.com/ethersphere/bee/v2/pkg/storer/internal/pinning" "github.com/ethersphere/bee/v2/pkg/storer/internal/reserve" "github.com/ethersphere/bee/v2/pkg/storer/internal/stampindex" "github.com/ethersphere/bee/v2/pkg/storer/internal/transaction" @@ -966,6 +967,139 @@ func TestReset(t *testing.T) { } } +// TestEvictRemovesPinnedContent checks that pinned chunks are protected from eviction. +func TestEvictRemovesPinnedContent(t *testing.T) { + t.Parallel() + + const ( + numChunks = 5 + numPinnedChunks = 3 + ) + + ctx := context.Background() + baseAddr := swarm.RandAddress(t) + ts := internal.NewInmemStorage() + + r, err := reserve.New( + baseAddr, + ts, + 0, + kademlia.NewTopologyDriver(), + log.Noop, + ) + if err != nil { + t.Fatal(err) + } + + batch := postagetesting.MustNewBatch() + + chunks := make([]swarm.Chunk, numChunks) + for i := 0; i < numChunks; i++ { + ch := chunk.GenerateTestRandomChunkAt(t, baseAddr, 0).WithStamp(postagetesting.MustNewBatchStamp(batch.ID)) + chunks[i] = ch + + err := r.Put(ctx, ch) + if err != nil { + t.Fatal(err) + } + } + + var pinningPutter internal.PutterCloserWithReference + err = ts.Run(ctx, func(store transaction.Store) error { + pinningPutter, err = pinstore.NewCollection(store.IndexStore()) + return err + }) + if err != nil { + t.Fatal(err) + } + + // Add chunks to pin collection + pinnedChunks := chunks[:numPinnedChunks] + for _, ch := range pinnedChunks { + err = ts.Run(ctx, func(s transaction.Store) error { + return pinningPutter.Put(ctx, s, ch) + }) + if err != nil { + t.Fatal(err) + } + } + err = ts.Run(ctx, func(s transaction.Store) error { + return pinningPutter.Close(s.IndexStore(), pinnedChunks[0].Address()) + }) + if err != nil { + t.Fatal(err) + } + + // evict all chunks from this batch - this should NOT remove pinned chunks + evicted, err := r.EvictBatchBin(ctx, batch.ID, numChunks, swarm.MaxBins) + if err != nil { + t.Fatal(err) + } + if evicted != numChunks { + t.Fatalf("expected %d evicted chunks, got %d", numChunks, evicted) + } + + uuids, err := pinstore.GetCollectionUUIDs(ts.IndexStore()) + if err != nil { + t.Fatal(err) + } + if len(uuids) != 1 { + t.Fatalf("expected exactly one pin collection, but found %d", len(uuids)) + } + + for i, ch := range chunks { + stampHash, err := ch.Stamp().Hash() + if err != nil { + t.Fatal(err) + } + + // Try to get the chunk from reserve, error is checked later + _, err = r.Get(ctx, ch.Address(), ch.Stamp().BatchID(), stampHash) + + // Also try to get chunk directly from chunkstore (like bzz/bytes endpoints do) + _, chunkStoreErr := ts.ChunkStore().Get(ctx, ch.Address()) + + pinned := false + for _, uuid := range uuids { + has, err := pinstore.IsChunkPinnedInCollection(ts.IndexStore(), ch.Address(), uuid) + if err != nil { + t.Fatal(err) + } + if has { + pinned = true + } + } + + if i < len(pinnedChunks) { + if pinned { + // This chunk is pinned, so it should NOT be accessible from reserve but SHOULD be accessible from chunkstore + if !errors.Is(err, storage.ErrNotFound) { + t.Errorf("Pinned chunk %s should have been evicted from reserve", ch.Address()) + } + if errors.Is(chunkStoreErr, storage.ErrNotFound) { + t.Errorf("Pinned chunk %s was deleted from chunkstore - should remain retrievable!", ch.Address()) + } else if chunkStoreErr != nil { + t.Fatal(chunkStoreErr) + } + } else { + t.Errorf("Chunk %s should be pinned", ch.Address()) + } + } else { // unpinned chunks + if !pinned { + // Unpinned chunks should be completely evicted (both reserve and chunkstore) + if !errors.Is(err, storage.ErrNotFound) { + t.Errorf("Unpinned chunk %s should have been evicted from reserve", ch.Address()) + } + if !errors.Is(chunkStoreErr, storage.ErrNotFound) { + t.Errorf("Unpinned chunk %s should have been evicted from chunkstore", ch.Address()) + } + } else { + t.Errorf("Chunk %s should not be pinned", ch.Address()) + } + } + } +} + func checkStore(t *testing.T, s storage.Reader, k storage.Key, gone bool) { t.Helper() h, err := s.Has(k) From 1a4f35141924ed8dafef7e55b17bc73bc08b9552 Mon Sep 17 00:00:00 2001 From: Akrem Chabchoub <121046693+akrem-chabchoub@users.noreply.github.com> Date: Sun, 5 Oct 2025 18:55:31 +0100 Subject: [PATCH 31/96] chore: bump golang to v1.25.1 and golangci-lint to v2 (#5231) --- .github/workflows/beekeeper.yml | 6 +- .github/workflows/codeql.yml | 2 +- .github/workflows/docs.yaml | 2 +- .github/workflows/go.yml | 24 ++-- .github/workflows/release.yaml | 4 +- .golangci.yml | 108 +++++++++--------- Dockerfile | 2 +- Dockerfile.dev | 2 +- Dockerfile.goreleaser | 2 +- Dockerfile.scratch | 2 +- Makefile | 2 +- cmd/bee/cmd/split.go | 4 +- go.mod | 6 +- pkg/accesscontrol/access.go | 2 +- pkg/accesscontrol/controller.go | 4 +- pkg/accesscontrol/controller_test.go | 2 +- pkg/accesscontrol/session.go | 2 +- pkg/api/api_test.go | 2 +- pkg/api/chequebook.go | 30 ++--- pkg/bmt/proof.go | 2 +- pkg/crypto/dh.go | 2 +- pkg/crypto/signer_test.go | 8 +- pkg/feeds/getter.go | 2 +- pkg/file/buffer.go | 5 +- pkg/file/pipeline/hashtrie/hashtrie.go | 8 +- pkg/file/utils.go | 6 +- pkg/jsonhttp/jsonhttptest/jsonhttptest.go | 9 +- pkg/keystore/file/key.go | 4 +- pkg/manifest/mantaray/marshal.go | 18 +-- pkg/manifest/mantaray/node.go | 8 +- pkg/manifest/mantaray/persist.go | 2 +- pkg/node/devnode.go | 2 +- pkg/node/node.go | 2 +- .../internal/blocklist/blocklist_test.go | 2 +- pkg/postage/postagecontract/contract_test.go | 20 ++-- pkg/settlement/swap/chequebook/factory.go | 2 +- pkg/shed/vector_uint64_test.go | 8 +- pkg/statestore/storeadapter/storeadapter.go | 10 +- pkg/status/status_test.go | 6 +- .../internal/chunkstore/chunkstore_test.go | 2 +- pkg/storer/internal/reserve/reserve_test.go | 7 +- pkg/storer/storer.go | 2 +- pkg/transaction/transaction.go | 2 +- pkg/util/testutil/racedetection/off.go | 1 - pkg/util/testutil/racedetection/on.go | 1 - 45 files changed, 179 insertions(+), 170 deletions(-) diff --git a/.github/workflows/beekeeper.yml b/.github/workflows/beekeeper.yml index 5006a520d2c..4c69fce7a6f 100644 --- a/.github/workflows/beekeeper.yml +++ b/.github/workflows/beekeeper.yml @@ -33,18 +33,18 @@ jobs: msg: ${{ steps.commit.outputs.msg }} steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 if: github.event.action != 'beekeeper' with: fetch-depth: 0 - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 if: github.event.action == 'beekeeper' with: fetch-depth: 0 ref: ${{ github.event.client_payload.ref }} - name: Setup Go - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: cache: false go-version-file: go.mod diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index dea9ed5ce5e..38402e52cda 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -40,7 +40,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 83097774128..fb41df6f301 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: fetch-depth: '0' - name: Check whether docs have changed diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index b705d804914..1ec8039073f 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -21,9 +21,9 @@ jobs: os: [ubuntu-latest, macos-latest, windows-latest] steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Setup Go - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: cache: true go-version-file: go.mod @@ -41,9 +41,9 @@ jobs: continue-on-error: ${{ github.ref == 'refs/heads/master' }} steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Setup Go - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: cache: true go-version-file: go.mod @@ -55,11 +55,11 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: fetch-depth: 0 - name: Setup Go - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: cache: false go-version-file: go.mod @@ -67,10 +67,10 @@ jobs: if: github.ref != 'refs/heads/master' uses: wagoid/commitlint-github-action@v5 - name: GolangCI-Lint - uses: golangci/golangci-lint-action@v6 + uses: golangci/golangci-lint-action@v8 with: skip-cache: false - version: v1.64.5 + version: v2.5.0 - name: Whitespace check run: make check-whitespace - name: go mod tidy check @@ -81,9 +81,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Setup Go - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: cache: false go-version-file: go.mod @@ -109,11 +109,11 @@ jobs: if: github.ref == 'refs/heads/master' steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: fetch-depth: 0 - name: Setup Go - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: cache: false go-version-file: go.mod diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 8d90e7c947d..877e4845a6a 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -12,11 +12,11 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: fetch-depth: 0 - name: Setup Go - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version-file: go.mod - name: Docker Hub and Quay Login diff --git a/.golangci.yml b/.golangci.yml index 4f07d6fa978..776f2575cbe 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,5 +1,4 @@ -run: - timeout: 10m +version: "2" linters: enable: - asciicheck @@ -7,19 +6,14 @@ linters: - copyloopvar - dogsled - durationcheck - - errcheck - errname - errorlint - forbidigo - gochecknoinits - goconst - - gofmt - goheader - goprintffuncname - - gosimple - - govet - importas - - ineffassign - misspell - nilerr - noctx @@ -27,49 +21,61 @@ linters: - prealloc - predeclared - promlinter - - staticcheck - thelper - - typecheck - unconvert - - unused - # - depguard disable temporary until this issue is resolved: https://github.com/golangci/golangci-lint/issues/3906 - -linters-settings: - govet: - enable-all: true - disable: - - fieldalignment ## temporally disabled - - shadow ## temporally disabled - goheader: - values: - regexp: - date: "20[1-2][0-9]" - template: |- - Copyright {{date}} The Swarm Authors. All rights reserved. - Use of this source code is governed by a BSD-style - license that can be found in the LICENSE file. - paralleltest: - # Ignore missing calls to `t.Parallel()` and only report incorrect uses of `t.Parallel()`. - ignore-missing: true -issues: - exclude-rules: - - linters: - - goheader - text: "go-ethereum Authors" ## disable check for other authors - - path: _test\.go - linters: - - goconst ## temporally disable goconst in test - - linters: - - forbidigo - path: cmd/bee/cmd - text: "use of `fmt.Print" ## allow fmt.Print in cmd directory - - linters: - - dogsled - path: pkg/api/(.+)_test\.go # temporally disable dogsled in api test files - - linters: - - dogsled - path: pkg/pushsync/(.+)_test\.go # temporally disable dogsled in pushsync test files - # temporally disable paralleltest in following packages - - linters: - - paralleltest - path: pkg/log + settings: + goheader: + values: + regexp: + date: 20[1-2][0-9] + template: |- + Copyright {{date}} The Swarm Authors. All rights reserved. + Use of this source code is governed by a BSD-style + license that can be found in the LICENSE file. + govet: + disable: + - fieldalignment + - shadow + enable-all: true + paralleltest: + ignore-missing: true + exclusions: + generated: lax + presets: + - comments + - common-false-positives + - legacy + - std-error-handling + rules: + - linters: + - goheader + text: go-ethereum Authors + - linters: + - goconst + path: _test\.go + - linters: + - forbidigo + path: cmd/bee/cmd + text: use of `fmt.Print + - linters: + - dogsled + path: pkg/api/(.+)_test\.go + - linters: + - dogsled + path: pkg/pushsync/(.+)_test\.go + - linters: + - paralleltest + path: pkg/log + paths: + - third_party$ + - builtin$ + - examples$ +formatters: + enable: + - gofmt + exclusions: + generated: lax + paths: + - third_party$ + - builtin$ + - examples$ diff --git a/Dockerfile b/Dockerfile index 0850b2ed8cf..01f7f2bad89 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.24 AS build +FROM golang:1.25 AS build WORKDIR /src # enable modules caching in separate layer diff --git a/Dockerfile.dev b/Dockerfile.dev index a417c710cbc..3cfae361aea 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -1,4 +1,4 @@ -FROM golang:1.24 AS build +FROM golang:1.25 AS build ARG REACHABILITY_OVERRIDE_PUBLIC=false ARG BATCHFACTOR_OVERRIDE_PUBLIC=5 diff --git a/Dockerfile.goreleaser b/Dockerfile.goreleaser index 2a62048228e..6772cb56963 100644 --- a/Dockerfile.goreleaser +++ b/Dockerfile.goreleaser @@ -1,4 +1,4 @@ -FROM debian:12.10-slim +FROM debian:12.12-slim ENV DEBIAN_FRONTEND=noninteractive diff --git a/Dockerfile.scratch b/Dockerfile.scratch index e11761aa326..1ecff0f7fbc 100644 --- a/Dockerfile.scratch +++ b/Dockerfile.scratch @@ -1,4 +1,4 @@ -FROM debian:12.10-slim +FROM debian:12.12-slim ENV DEBIAN_FRONTEND=noninteractive diff --git a/Makefile b/Makefile index 01568f1136b..047f5d4f8be 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ GO ?= go GOBIN ?= $$($(GO) env GOPATH)/bin GOLANGCI_LINT ?= $(GOBIN)/golangci-lint -GOLANGCI_LINT_VERSION ?= v1.64.5 +GOLANGCI_LINT_VERSION ?= v2.5.0 GOGOPROTOBUF ?= protoc-gen-gogofaster GOGOPROTOBUF_VERSION ?= v1.3.1 BEEKEEPER_INSTALL_DIR ?= $(GOBIN) diff --git a/cmd/bee/cmd/split.go b/cmd/bee/cmd/split.go index ebe798e878d..112b32ed2fa 100644 --- a/cmd/bee/cmd/split.go +++ b/cmd/bee/cmd/split.go @@ -116,13 +116,13 @@ func splitRefs(cmd *cobra.Command) { } logger.Debug("write root", "hash", rootRef) - _, err = writer.WriteString(fmt.Sprintf("%s\n", rootRef)) + _, err = fmt.Fprintf(writer, "%s\n", rootRef) if err != nil { return fmt.Errorf("write root hash: %w", err) } for _, ref := range refs { logger.Debug("write chunk", "hash", ref) - _, err = writer.WriteString(fmt.Sprintf("%s\n", ref)) + _, err = fmt.Fprintf(writer, "%s\n", ref) if err != nil { return fmt.Errorf("write chunk address: %w", err) } diff --git a/go.mod b/go.mod index dfd5fbd4a5e..eccad953b87 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,8 @@ module github.com/ethersphere/bee/v2 -go 1.24.0 +go 1.25 -toolchain go1.24.2 +toolchain go1.25.1 require ( contrib.go.opencensus.io/exporter/prometheus v0.4.2 @@ -46,7 +46,6 @@ require ( go.uber.org/atomic v1.11.0 go.uber.org/goleak v1.3.0 golang.org/x/crypto v0.36.0 - golang.org/x/net v0.38.0 golang.org/x/sync v0.12.0 golang.org/x/sys v0.31.0 golang.org/x/term v0.30.0 @@ -170,6 +169,7 @@ require ( go.uber.org/zap v1.27.0 // indirect golang.org/x/exp v0.0.0-20240213143201-ec583247a57a // indirect golang.org/x/mod v0.22.0 // indirect + golang.org/x/net v0.38.0 // indirect golang.org/x/text v0.23.0 // indirect golang.org/x/tools v0.29.0 // indirect google.golang.org/protobuf v1.36.1 // indirect diff --git a/pkg/accesscontrol/access.go b/pkg/accesscontrol/access.go index 0b7f9a094ac..71ddd3500c5 100644 --- a/pkg/accesscontrol/access.go +++ b/pkg/accesscontrol/access.go @@ -129,7 +129,7 @@ func (al *ActLogic) getAccessKey(ctx context.Context, storage kvs.KeyValueStore, // Generate lookup key and access key decryption key for a given public key. func (al *ActLogic) getKeys(publicKey *ecdsa.PublicKey) ([]byte, []byte, error) { nonces := [][]byte{zeroByteArray, oneByteArray} - keys, err := al.Session.Key(publicKey, nonces) + keys, err := al.Key(publicKey, nonces) if len(keys) != len(nonces) { return nil, nil, err } diff --git a/pkg/accesscontrol/controller.go b/pkg/accesscontrol/controller.go index 804c307ce8c..cc3c8ae6fd7 100644 --- a/pkg/accesscontrol/controller.go +++ b/pkg/accesscontrol/controller.go @@ -261,7 +261,7 @@ func (c *ControllerStruct) getGranteeList(ctx context.Context, ls file.LoadSaver } func (c *ControllerStruct) encryptRefForPublisher(publisherPubKey *ecdsa.PublicKey, ref swarm.Address) (swarm.Address, error) { - keys, err := c.access.Session.Key(publisherPubKey, [][]byte{oneByteArray}) + keys, err := c.access.Key(publisherPubKey, [][]byte{oneByteArray}) if err != nil { return swarm.ZeroAddress, err } @@ -275,7 +275,7 @@ func (c *ControllerStruct) encryptRefForPublisher(publisherPubKey *ecdsa.PublicK } func (c *ControllerStruct) decryptRefForPublisher(publisherPubKey *ecdsa.PublicKey, encryptedRef swarm.Address) (swarm.Address, error) { - keys, err := c.access.Session.Key(publisherPubKey, [][]byte{oneByteArray}) + keys, err := c.access.Key(publisherPubKey, [][]byte{oneByteArray}) if err != nil { return swarm.ZeroAddress, err } diff --git a/pkg/accesscontrol/controller_test.go b/pkg/accesscontrol/controller_test.go index 915d82d7d8b..1fa3da681ce 100644 --- a/pkg/accesscontrol/controller_test.go +++ b/pkg/accesscontrol/controller_test.go @@ -178,7 +178,7 @@ func TestController_UpdateHandler(t *testing.T) { publisher := getPrivKey(1) diffieHellman := accesscontrol.NewDefaultSession(publisher) al := accesscontrol.NewLogic(diffieHellman) - keys, err := al.Session.Key(&publisher.PublicKey, [][]byte{{1}}) + keys, err := al.Key(&publisher.PublicKey, [][]byte{{1}}) assertNoError(t, "Session key", err) refCipher := encryption.New(keys[0], 0, 0, sha3.NewLegacyKeccak256) ls := createLs() diff --git a/pkg/accesscontrol/session.go b/pkg/accesscontrol/session.go index a571337c17d..f3996264970 100644 --- a/pkg/accesscontrol/session.go +++ b/pkg/accesscontrol/session.go @@ -37,7 +37,7 @@ func (s *SessionStruct) Key(publicKey *ecdsa.PublicKey, nonces [][]byte) ([][]by if publicKey == nil { return nil, ErrInvalidPublicKey } - x, y := publicKey.Curve.ScalarMult(publicKey.X, publicKey.Y, s.key.D.Bytes()) + x, y := publicKey.ScalarMult(publicKey.X, publicKey.Y, s.key.D.Bytes()) if x == nil || y == nil { return nil, ErrSecretKeyInfinity } diff --git a/pkg/api/api_test.go b/pkg/api/api_test.go index 986bc663d3e..4dd7aa1d3f7 100644 --- a/pkg/api/api_test.go +++ b/pkg/api/api_test.go @@ -266,7 +266,7 @@ func newTestServer(t *testing.T, o testServerOptions) (*http.Client, *websocket. transport = transport.Clone() // always dial to the server address, regardless of the url host and port transport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { - return net.Dial(network, ts.Listener.Addr().String()) + return (&net.Dialer{}).DialContext(ctx, network, ts.Listener.Addr().String()) } return transport.RoundTrip(r) }), diff --git a/pkg/api/chequebook.go b/pkg/api/chequebook.go index 219f9479089..cb82e72edfd 100644 --- a/pkg/api/chequebook.go +++ b/pkg/api/chequebook.go @@ -117,9 +117,9 @@ func (s *Service) chequebookLastPeerHandler(w http.ResponseWriter, r *http.Reque } if err == nil { lastSentResponse = &chequebookLastChequePeerResponse{ - Beneficiary: lastSent.Cheque.Beneficiary.String(), - Chequebook: lastSent.Cheque.Chequebook.String(), - Payout: bigint.Wrap(lastSent.Cheque.CumulativePayout), + Beneficiary: lastSent.Beneficiary.String(), + Chequebook: lastSent.Chequebook.String(), + Payout: bigint.Wrap(lastSent.CumulativePayout), } } @@ -133,9 +133,9 @@ func (s *Service) chequebookLastPeerHandler(w http.ResponseWriter, r *http.Reque } if err == nil { lastReceivedResponse = &chequebookLastChequePeerResponse{ - Beneficiary: lastReceived.Cheque.Beneficiary.String(), - Chequebook: lastReceived.Cheque.Chequebook.String(), - Payout: bigint.Wrap(lastReceived.Cheque.CumulativePayout), + Beneficiary: lastReceived.Beneficiary.String(), + Chequebook: lastReceived.Chequebook.String(), + Payout: bigint.Wrap(lastReceived.CumulativePayout), } } @@ -178,9 +178,9 @@ func (s *Service) chequebookAllLastHandler(w http.ResponseWriter, _ *http.Reques lcr[i] = chequebookLastChequesPeerResponse{ Peer: i, LastSent: &chequebookLastChequePeerResponse{ - Beneficiary: j.Cheque.Beneficiary.String(), - Chequebook: j.Cheque.Chequebook.String(), - Payout: bigint.Wrap(j.Cheque.CumulativePayout), + Beneficiary: j.Beneficiary.String(), + Chequebook: j.Chequebook.String(), + Payout: bigint.Wrap(j.CumulativePayout), }, LastReceived: nil, } @@ -189,9 +189,9 @@ func (s *Service) chequebookAllLastHandler(w http.ResponseWriter, _ *http.Reques if _, ok := lcr[i]; ok { t := lcr[i] t.LastReceived = &chequebookLastChequePeerResponse{ - Beneficiary: j.Cheque.Beneficiary.String(), - Chequebook: j.Cheque.Chequebook.String(), - Payout: bigint.Wrap(j.Cheque.CumulativePayout), + Beneficiary: j.Beneficiary.String(), + Chequebook: j.Chequebook.String(), + Payout: bigint.Wrap(j.CumulativePayout), } lcr[i] = t } else { @@ -199,9 +199,9 @@ func (s *Service) chequebookAllLastHandler(w http.ResponseWriter, _ *http.Reques Peer: i, LastSent: nil, LastReceived: &chequebookLastChequePeerResponse{ - Beneficiary: j.Cheque.Beneficiary.String(), - Chequebook: j.Cheque.Chequebook.String(), - Payout: bigint.Wrap(j.Cheque.CumulativePayout), + Beneficiary: j.Beneficiary.String(), + Chequebook: j.Chequebook.String(), + Payout: bigint.Wrap(j.CumulativePayout), }, } } diff --git a/pkg/bmt/proof.go b/pkg/bmt/proof.go index 481a50d87e5..b08017b43cd 100644 --- a/pkg/bmt/proof.go +++ b/pkg/bmt/proof.go @@ -20,7 +20,7 @@ type Proof struct { // Hash overrides base hash function of Hasher to fill buffer with zeros until chunk length func (p Prover) Hash(b []byte) ([]byte, error) { for i := p.size; i < p.maxSize; i += len(zerosection) { - _, err := p.Hasher.Write(zerosection) + _, err := p.Write(zerosection) if err != nil { return nil, err } diff --git a/pkg/crypto/dh.go b/pkg/crypto/dh.go index 2d5b4e695fc..f3ea41facc4 100644 --- a/pkg/crypto/dh.go +++ b/pkg/crypto/dh.go @@ -29,7 +29,7 @@ func NewDH(key *ecdsa.PrivateKey) DH { // safety warning: this method is not meant to be exposed as it does not validate private and public keys // are on the same curve func (dh *defaultDH) SharedKey(pub *ecdsa.PublicKey, salt []byte) ([]byte, error) { - x, _ := pub.Curve.ScalarMult(pub.X, pub.Y, dh.key.D.Bytes()) + x, _ := pub.ScalarMult(pub.X, pub.Y, dh.key.D.Bytes()) if x == nil { return nil, errors.New("shared secret is point at infinity") } diff --git a/pkg/crypto/signer_test.go b/pkg/crypto/signer_test.go index b3b09ae04af..6d6690ec27e 100644 --- a/pkg/crypto/signer_test.go +++ b/pkg/crypto/signer_test.go @@ -41,7 +41,7 @@ func TestDefaultSigner(t *testing.T) { t.Fatal(err) } - if pubKey.X.Cmp(privKey.PublicKey.X) != 0 || pubKey.Y.Cmp(privKey.PublicKey.Y) != 0 { + if pubKey.X.Cmp(privKey.X) != 0 || pubKey.Y.Cmp(privKey.Y) != 0 { t.Fatalf("wanted %v but got %v", pubKey, &privKey.PublicKey) } }) @@ -54,7 +54,7 @@ func TestDefaultSigner(t *testing.T) { t.Fatal(err) } - if pubKey.X.Cmp(privKey.PublicKey.X) == 0 && pubKey.Y.Cmp(privKey.PublicKey.Y) == 0 { + if pubKey.X.Cmp(privKey.X) == 0 && pubKey.Y.Cmp(privKey.Y) == 0 { t.Fatal("expected different public key") } }) @@ -228,11 +228,11 @@ func TestRecoverEIP712(t *testing.T) { t.Fatal(err) } - if privKey.PublicKey.X.Cmp(pubKey.X) != 0 { + if privKey.X.Cmp(pubKey.X) != 0 { t.Fatalf("recovered wrong public key. wanted %x, got %x", privKey.PublicKey, pubKey) } - if privKey.PublicKey.Y.Cmp(pubKey.Y) != 0 { + if privKey.Y.Cmp(pubKey.Y) != 0 { t.Fatalf("recovered wrong public key. wanted %x, got %x", privKey.PublicKey, pubKey) } } diff --git a/pkg/feeds/getter.go b/pkg/feeds/getter.go index d0ab55da024..2bdf509e282 100644 --- a/pkg/feeds/getter.go +++ b/pkg/feeds/getter.go @@ -93,7 +93,7 @@ func FromChunk(ch swarm.Chunk) (swarm.Chunk, error) { // legacyPayload returns back the referenced chunk and datetime from the legacy feed payload func legacyPayload(wrappedChunk swarm.Chunk) (swarm.Address, error) { cacData := wrappedChunk.Data() - if !(len(cacData) == 16+swarm.HashSize || len(cacData) == 16+swarm.HashSize*2) { + if len(cacData) != 16+swarm.HashSize && len(cacData) != 16+swarm.HashSize*2 { return swarm.ZeroAddress, ErrNotLegacyPayload } diff --git a/pkg/file/buffer.go b/pkg/file/buffer.go index 976eca20fa0..aaf36f535f3 100644 --- a/pkg/file/buffer.go +++ b/pkg/file/buffer.go @@ -42,10 +42,7 @@ func (c *ChunkPipe) Read(b []byte) (int, error) { func (c *ChunkPipe) Write(b []byte) (int, error) { nw := 0 - for { - if nw >= len(b) { - break - } + for nw < len(b) { copied := copy(c.data[c.cursor:], b[nw:]) c.cursor += copied diff --git a/pkg/file/pipeline/hashtrie/hashtrie.go b/pkg/file/pipeline/hashtrie/hashtrie.go index 1aff8c0df66..9c2d17fb5fc 100644 --- a/pkg/file/pipeline/hashtrie/hashtrie.go +++ b/pkg/file/pipeline/hashtrie/hashtrie.go @@ -198,11 +198,11 @@ func (h *hashTrieWriter) wrapFullLevel(level int) error { func (h *hashTrieWriter) Sum() ([]byte, error) { for i := 1; i < maxLevel; i++ { l := h.chunkCounters[i] - switch { - case l == 0: + switch l { + case 0: // level empty, continue to the next. continue - case l == h.maxChildrenChunks: + case h.maxChildrenChunks: // this case is possible and necessary due to the carry over // in the next switch case statement. normal writes done // through writeToLevel will automatically wrap a full level. @@ -211,7 +211,7 @@ func (h *hashTrieWriter) Sum() ([]byte, error) { if err != nil { return nil, err } - case l == 1: + case 1: // this cursor assignment basically means: // take the hash|span|key from this level, and append it to // the data of the next level. you may wonder how this works: diff --git a/pkg/file/utils.go b/pkg/file/utils.go index ce1f94d372a..b0e3071f98d 100644 --- a/pkg/file/utils.go +++ b/pkg/file/utils.go @@ -66,10 +66,8 @@ func ReferenceCount(span uint64, level redundancy.Level, encrytedChunk bool) (in ) // search for branch level big enough to include span branchLevel := 1 - for { - if branchSize >= span { - break - } + for branchSize < span { + branchSize *= branching branchLevel++ } diff --git a/pkg/jsonhttp/jsonhttptest/jsonhttptest.go b/pkg/jsonhttp/jsonhttptest/jsonhttptest.go index 1c436ec9373..7902a566dc9 100644 --- a/pkg/jsonhttp/jsonhttptest/jsonhttptest.go +++ b/pkg/jsonhttp/jsonhttptest/jsonhttptest.go @@ -36,14 +36,15 @@ func Request(tb testing.TB, client *http.Client, method, url string, responseCod } } - req, err := http.NewRequest(method, url, o.requestBody) + ctx := o.ctx + if ctx == nil { + ctx = context.Background() + } + req, err := http.NewRequestWithContext(ctx, method, url, o.requestBody) if err != nil { tb.Fatal(err) } req.Header = o.requestHeaders - if o.ctx != nil { - req = req.WithContext(o.ctx) - } resp, err := client.Do(req) if err != nil { tb.Fatal(err) diff --git a/pkg/keystore/file/key.go b/pkg/keystore/file/key.go index dcbe22612f5..5844b0dd1ef 100644 --- a/pkg/keystore/file/key.go +++ b/pkg/keystore/file/key.go @@ -75,7 +75,7 @@ func encryptKey(k *ecdsa.PrivateKey, password string, edg keystore.EDG) ([]byte, return nil, err } var addr []byte - switch k.PublicKey.Curve { + switch k.Curve { case btcec.S256(): a, err := crypto.NewEthereumAddress(k.PublicKey) if err != nil { @@ -89,7 +89,7 @@ func encryptKey(k *ecdsa.PrivateKey, password string, edg keystore.EDG) ([]byte, } addr = privKey.PublicKey().Bytes() default: - return nil, fmt.Errorf("unsupported curve: %v", k.PublicKey.Curve) + return nil, fmt.Errorf("unsupported curve: %v", k.Curve) } return json.Marshal(encryptedKey{ Address: hex.EncodeToString(addr), diff --git a/pkg/manifest/mantaray/marshal.go b/pkg/manifest/mantaray/marshal.go index 0129312ff9f..f5f7edabc8f 100644 --- a/pkg/manifest/mantaray/marshal.go +++ b/pkg/manifest/mantaray/marshal.go @@ -12,6 +12,7 @@ import ( "encoding/json" "errors" "fmt" + "math" ) const ( @@ -341,7 +342,7 @@ func (f *fork) fromBytes(b []byte) error { f.prefix = b[nodeForkHeaderSize : nodeForkHeaderSize+prefixLen] f.Node = NewNodeRef(b[nodeForkPreReferenceSize:]) - f.Node.nodeType = nodeType + f.nodeType = nodeType return nil } @@ -356,7 +357,7 @@ func (f *fork) fromBytes02(b []byte, refBytesSize, metadataBytesSize int) error f.prefix = b[nodeForkHeaderSize : nodeForkHeaderSize+prefixLen] f.Node = NewNodeRef(b[nodeForkPreReferenceSize : nodeForkPreReferenceSize+refBytesSize]) - f.Node.nodeType = nodeType + f.nodeType = nodeType if metadataBytesSize > 0 { metadataBytes := b[nodeForkPreReferenceSize+refBytesSize+nodeForkMetadataBytesSize:] @@ -368,7 +369,7 @@ func (f *fork) fromBytes02(b []byte, refBytesSize, metadataBytesSize int) error return err } - f.Node.metadata = metadata + f.metadata = metadata } return nil @@ -381,7 +382,7 @@ func (f *fork) bytes() (b []byte, err error) { err = fmt.Errorf("node reference size > 256: %d", len(r)) return } - b = append(b, f.Node.nodeType, uint8(len(f.prefix))) + b = append(b, f.nodeType, uint8(len(f.prefix))) prefixBytes := make([]byte, nodePrefixMaxSize) copy(prefixBytes, f.prefix) @@ -391,13 +392,16 @@ func (f *fork) bytes() (b []byte, err error) { copy(refBytes, r) b = append(b, refBytes...) - if f.Node.IsWithMetadataType() { + if f.IsWithMetadataType() { // using JSON encoding for metadata - metadataJSONBytes, err1 := json.Marshal(f.Node.metadata) + metadataJSONBytes, err1 := json.Marshal(f.metadata) if err1 != nil { return b, err1 } + if len(metadataJSONBytes) > math.MaxInt-nodeForkMetadataBytesSize { + return nil, fmt.Errorf("metadata size overflow: %d", len(metadataJSONBytes)) + } metadataJSONBytesSizeWithSize := len(metadataJSONBytes) + nodeForkMetadataBytesSize // pad JSON bytes if necessary @@ -435,7 +439,7 @@ func (f *fork) bytes() (b []byte, err error) { var refBytes = nodeRefBytes func nodeRefBytes(f *fork) []byte { - return f.Node.ref + return f.ref } // encryptDecrypt runs a XOR encryption on the input bytes, encrypting it if it diff --git a/pkg/manifest/mantaray/node.go b/pkg/manifest/mantaray/node.go index 2e2754f200b..88d4a2cbc6a 100644 --- a/pkg/manifest/mantaray/node.go +++ b/pkg/manifest/mantaray/node.go @@ -166,7 +166,7 @@ func (n *Node) LookupNode(ctx context.Context, path []byte, l Loader) (*Node, er } c := common(f.prefix, path) if len(c) == len(f.prefix) { - return f.Node.LookupNode(ctx, path[len(c):], l) + return f.LookupNode(ctx, path[len(c):], l) } return nil, notFound(path) } @@ -259,7 +259,7 @@ func (n *Node) Add(ctx context.Context, path, entry []byte, metadata map[string] nn.SetObfuscationKey(n.obfuscationKey) } nn.refBytesSize = n.refBytesSize - f.Node.updateIsWithPathSeparator(rest) + f.updateIsWithPathSeparator(rest) nn.forks[rest[0]] = &fork{rest, f.Node} nn.makeEdge() // if common path is full path new node is value type @@ -319,7 +319,7 @@ func (n *Node) Remove(ctx context.Context, path []byte, ls LoadSaver) error { delete(n.forks, path[0]) return nil } - return f.Node.Remove(ctx, rest, ls) + return f.Remove(ctx, rest, ls) } func common(a, b []byte) (c []byte) { @@ -350,7 +350,7 @@ func (n *Node) HasPrefix(ctx context.Context, path []byte, l Loader) (bool, erro } c := common(f.prefix, path) if len(c) == len(f.prefix) { - return f.Node.HasPrefix(ctx, path[len(c):], l) + return f.HasPrefix(ctx, path[len(c):], l) } if bytes.HasPrefix(f.prefix, path) { return true, nil diff --git a/pkg/manifest/mantaray/persist.go b/pkg/manifest/mantaray/persist.go index 8ac812e50d8..259785d803c 100644 --- a/pkg/manifest/mantaray/persist.go +++ b/pkg/manifest/mantaray/persist.go @@ -71,7 +71,7 @@ func (n *Node) save(ctx context.Context, s Saver) error { eg, ectx := errgroup.WithContext(ctx) for _, f := range n.forks { eg.Go(func() error { - return f.Node.save(ectx, s) + return f.save(ectx, s) }) } if err := eg.Wait(); err != nil { diff --git a/pkg/node/devnode.go b/pkg/node/devnode.go index 9ff5363c455..acf69ce026f 100644 --- a/pkg/node/devnode.go +++ b/pkg/node/devnode.go @@ -373,7 +373,7 @@ func NewDevBee(logger log.Logger, o *DevOptions) (b *DevBee, err error) { apiService.SetP2P(p2ps) apiService.SetSwarmAddress(&swarmAddress) - apiListener, err := net.Listen("tcp", o.APIAddr) + apiListener, err := (&net.ListenConfig{}).Listen(context.Background(), "tcp", o.APIAddr) if err != nil { return nil, fmt.Errorf("api listener: %w", err) } diff --git a/pkg/node/node.go b/pkg/node/node.go index 67d185a55c5..163b599e651 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -448,7 +448,7 @@ func NewBee( runtime.SetBlockProfileRate(1) } - apiListener, err := net.Listen("tcp", o.APIAddr) + apiListener, err := (&net.ListenConfig{}).Listen(ctx, "tcp", o.APIAddr) if err != nil { return nil, fmt.Errorf("api listener: %w", err) } diff --git a/pkg/p2p/libp2p/internal/blocklist/blocklist_test.go b/pkg/p2p/libp2p/internal/blocklist/blocklist_test.go index 74a5ba98258..da80a9ebea1 100644 --- a/pkg/p2p/libp2p/internal/blocklist/blocklist_test.go +++ b/pkg/p2p/libp2p/internal/blocklist/blocklist_test.go @@ -111,7 +111,7 @@ func TestPeers(t *testing.T) { func isIn(p swarm.Address, peers []p2p.BlockListedPeer, reason string, f bool) bool { for _, v := range peers { - if v.Address.Equal(p) && v.Reason == reason && v.Peer.FullNode == f { + if v.Address.Equal(p) && v.Reason == reason && v.FullNode == f { return true } } diff --git a/pkg/postage/postagecontract/contract_test.go b/pkg/postage/postagecontract/contract_test.go index 756bbe24a9d..80da7a8ab0d 100644 --- a/pkg/postage/postagecontract/contract_test.go +++ b/pkg/postage/postagecontract/contract_test.go @@ -76,9 +76,10 @@ func TestCreateBatch(t *testing.T) { bzzTokenAddress, transactionMock.New( transactionMock.WithSendFunc(func(ctx context.Context, request *transaction.TxRequest, boost int) (txHash common.Hash, err error) { - if *request.To == bzzTokenAddress { + switch *request.To { + case bzzTokenAddress: return txHashApprove, nil - } else if *request.To == postageStampAddress { + case postageStampAddress: if bytes.Equal(expectedCallDataForExpireLimitedBatches[:32], request.Data[:32]) { return txHashApprove, nil } @@ -90,11 +91,12 @@ func TestCreateBatch(t *testing.T) { return common.Hash{}, errors.New("sent to wrong contract") }), transactionMock.WithWaitForReceiptFunc(func(ctx context.Context, txHash common.Hash) (receipt *types.Receipt, err error) { - if txHash == txHashApprove { + switch txHash { + case txHashApprove: return &types.Receipt{ Status: 1, }, nil - } else if txHash == txHashCreate { + case txHashCreate: return &types.Receipt{ Logs: []*types.Log{ newCreateEvent(postageStampAddress, batchID), @@ -321,9 +323,10 @@ func TestTopUpBatch(t *testing.T) { bzzTokenAddress, transactionMock.New( transactionMock.WithSendFunc(func(ctx context.Context, request *transaction.TxRequest, boost int) (txHash common.Hash, err error) { - if *request.To == bzzTokenAddress { + switch *request.To { + case bzzTokenAddress: return txHashApprove, nil - } else if *request.To == postageStampAddress { + case postageStampAddress: if !bytes.Equal(expectedCallData[:64], request.Data[:64]) { return common.Hash{}, fmt.Errorf("got wrong call data. wanted %x, got %x", expectedCallData, request.Data) } @@ -332,11 +335,12 @@ func TestTopUpBatch(t *testing.T) { return common.Hash{}, errors.New("sent to wrong contract") }), transactionMock.WithWaitForReceiptFunc(func(ctx context.Context, txHash common.Hash) (receipt *types.Receipt, err error) { - if txHash == txHashApprove { + switch txHash { + case txHashApprove: return &types.Receipt{ Status: 1, }, nil - } else if txHash == txHashTopup { + case txHashTopup: return &types.Receipt{ Logs: []*types.Log{ newTopUpEvent(postageStampAddress, batch), diff --git a/pkg/settlement/swap/chequebook/factory.go b/pkg/settlement/swap/chequebook/factory.go index 53457679d2e..f548643bac1 100644 --- a/pkg/settlement/swap/chequebook/factory.go +++ b/pkg/settlement/swap/chequebook/factory.go @@ -9,13 +9,13 @@ import ( "fmt" "math/big" + "context" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" "github.com/ethersphere/bee/v2/pkg/sctx" "github.com/ethersphere/bee/v2/pkg/transaction" "github.com/ethersphere/bee/v2/pkg/util/abiutil" "github.com/ethersphere/go-sw3-abi/sw3abi" - "golang.org/x/net/context" ) var ( diff --git a/pkg/shed/vector_uint64_test.go b/pkg/shed/vector_uint64_test.go index 2d602a034c0..4eeaf1cdaf3 100644 --- a/pkg/shed/vector_uint64_test.go +++ b/pkg/shed/vector_uint64_test.go @@ -56,7 +56,7 @@ func TestUint64Vector(t *testing.T) { } for _, index := range []uint64{0, 1, 2, 5, 100} { - var want uint64 = 42 + index + var want = 42 + index err = bins.Put(index, want) if err != nil { t.Fatal(err) @@ -70,7 +70,7 @@ func TestUint64Vector(t *testing.T) { } t.Run("overwrite", func(t *testing.T) { - var want uint64 = 84 + index + var want = 84 + index err = bins.Put(index, want) if err != nil { t.Fatal(err) @@ -97,7 +97,7 @@ func TestUint64Vector(t *testing.T) { for _, index := range []uint64{0, 1, 2, 3, 5, 10} { batch := new(leveldb.Batch) - var want uint64 = 43 + index + var want = 43 + index bins.PutInBatch(batch, index, want) err = db.WriteBatch(batch) if err != nil { @@ -113,7 +113,7 @@ func TestUint64Vector(t *testing.T) { t.Run("overwrite", func(t *testing.T) { batch := new(leveldb.Batch) - var want uint64 = 85 + index + var want = 85 + index bins.PutInBatch(batch, index, want) err = db.WriteBatch(batch) if err != nil { diff --git a/pkg/statestore/storeadapter/storeadapter.go b/pkg/statestore/storeadapter/storeadapter.go index 2f05f1f31c7..1deda6c6cd3 100644 --- a/pkg/statestore/storeadapter/storeadapter.go +++ b/pkg/statestore/storeadapter/storeadapter.go @@ -105,11 +105,11 @@ type rawItem struct { // Marshal implements Item interface. func (ri *rawItem) Marshal() ([]byte, error) { - if ri == nil || ri.proxyItem == nil || ri.proxyItem.obj == nil { + if ri == nil || ri.proxyItem == nil || ri.obj == nil { return nil, nil } - if buf, ok := ri.proxyItem.obj.([]byte); ok { + if buf, ok := ri.obj.([]byte); ok { return buf, nil } @@ -118,12 +118,12 @@ func (ri *rawItem) Marshal() ([]byte, error) { // Unmarshal implements Item interface. func (ri *rawItem) Unmarshal(data []byte) error { - if ri == nil || ri.proxyItem == nil || ri.proxyItem.obj == nil || len(data) == 0 { + if ri == nil || ri.proxyItem == nil || ri.obj == nil || len(data) == 0 { return nil } - if buf, ok := ri.proxyItem.obj.([]byte); ok { - ri.proxyItem.obj = append(buf[:0], data...) + if buf, ok := ri.obj.([]byte); ok { + ri.obj = append(buf[:0], data...) return nil } diff --git a/pkg/status/status_test.go b/pkg/status/status_test.go index 09213ec71d4..2b3f9383fe3 100644 --- a/pkg/status/status_test.go +++ b/pkg/status/status_test.go @@ -249,12 +249,12 @@ type statusSnapshotMock struct { *pb.Snapshot } -func (m *statusSnapshotMock) SyncRate() float64 { return m.Snapshot.PullsyncRate } +func (m *statusSnapshotMock) SyncRate() float64 { return m.PullsyncRate } func (m *statusSnapshotMock) ReserveSize() int { return int(m.Snapshot.ReserveSize) } func (m *statusSnapshotMock) StorageRadius() uint8 { return uint8(m.Snapshot.StorageRadius) } -func (m *statusSnapshotMock) Commitment() (uint64, error) { return m.Snapshot.BatchCommitment, nil } +func (m *statusSnapshotMock) Commitment() (uint64, error) { return m.BatchCommitment, nil } func (m *statusSnapshotMock) GetChainState() *postage.ChainState { - return &postage.ChainState{Block: m.Snapshot.LastSyncedBlock} + return &postage.ChainState{Block: m.LastSyncedBlock} } func (m *statusSnapshotMock) ReserveSizeWithinRadius() uint64 { return m.Snapshot.ReserveSizeWithinRadius diff --git a/pkg/storer/internal/chunkstore/chunkstore_test.go b/pkg/storer/internal/chunkstore/chunkstore_test.go index 6787cf826ab..970e92df9b8 100644 --- a/pkg/storer/internal/chunkstore/chunkstore_test.go +++ b/pkg/storer/internal/chunkstore/chunkstore_test.go @@ -109,7 +109,7 @@ type memFS struct { } func (m *memFS) Open(path string) (fs.File, error) { - return m.Fs.OpenFile(path, os.O_RDWR|os.O_CREATE, 0o644) + return m.OpenFile(path, os.O_RDWR|os.O_CREATE, 0o644) } func TestChunkStore(t *testing.T) { diff --git a/pkg/storer/internal/reserve/reserve_test.go b/pkg/storer/internal/reserve/reserve_test.go index c6d682a7171..4633c140385 100644 --- a/pkg/storer/internal/reserve/reserve_test.go +++ b/pkg/storer/internal/reserve/reserve_test.go @@ -118,11 +118,12 @@ func TestReserveChunkType(t *testing.T) { Factory: func() storage.Item { return &reserve.ChunkBinItem{} }, }, func(res storage.Result) (bool, error) { item := res.Entry.(*reserve.ChunkBinItem) - if item.ChunkType == swarm.ChunkTypeContentAddressed { + switch item.ChunkType { + case swarm.ChunkTypeContentAddressed: storedChunksCA-- - } else if item.ChunkType == swarm.ChunkTypeSingleOwner { + case swarm.ChunkTypeSingleOwner: storedChunksSO-- - } else { + default: t.Fatalf("unexpected chunk type: %d", item.ChunkType) } return false, nil diff --git a/pkg/storer/storer.go b/pkg/storer/storer.go index 76b35f198ba..e3318c9b278 100644 --- a/pkg/storer/storer.go +++ b/pkg/storer/storer.go @@ -189,7 +189,7 @@ type memFS struct { } func (m *memFS) Open(path string) (fs.File, error) { - return m.Fs.OpenFile(path, os.O_RDWR|os.O_CREATE, 0o644) + return m.OpenFile(path, os.O_RDWR|os.O_CREATE, 0o644) } type dirFS struct { diff --git a/pkg/transaction/transaction.go b/pkg/transaction/transaction.go index da71a2150e3..2f2405dd432 100644 --- a/pkg/transaction/transaction.go +++ b/pkg/transaction/transaction.go @@ -14,6 +14,7 @@ import ( "sync" "time" + "context" "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" @@ -23,7 +24,6 @@ import ( "github.com/ethersphere/bee/v2/pkg/log" "github.com/ethersphere/bee/v2/pkg/sctx" "github.com/ethersphere/bee/v2/pkg/storage" - "golang.org/x/net/context" ) // loggerName is the tree path name of the logger for this package. diff --git a/pkg/util/testutil/racedetection/off.go b/pkg/util/testutil/racedetection/off.go index d57125bfd03..cd7bfe36c55 100644 --- a/pkg/util/testutil/racedetection/off.go +++ b/pkg/util/testutil/racedetection/off.go @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build !race -// +build !race package racedetection diff --git a/pkg/util/testutil/racedetection/on.go b/pkg/util/testutil/racedetection/on.go index 92438ecfdc3..428ebb16800 100644 --- a/pkg/util/testutil/racedetection/on.go +++ b/pkg/util/testutil/racedetection/on.go @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build race -// +build race package racedetection From a74befdaf70420dd0bb7119451267a0db408a2e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jano=C5=A1=20Gulja=C5=A1?= Date: Mon, 6 Oct 2025 14:41:41 +0200 Subject: [PATCH 32/96] chore: modernize codebase for go 1.25 (#5236) --- cmd/bee/cmd/db_test.go | 10 +- pkg/accounting/accounting.go | 20 +-- pkg/accounting/accounting_test.go | 10 +- pkg/api/api.go | 4 +- pkg/api/bzz_test.go | 11 +- pkg/api/chunk_stream_test.go | 2 +- pkg/api/export_test.go | 2 +- pkg/api/logger_test.go | 4 +- pkg/api/postage_test.go | 2 +- pkg/api/tag.go | 2 +- pkg/api/util.go | 2 +- pkg/api/util_test.go | 16 +-- pkg/bitvector/bitvector_test.go | 6 +- pkg/blocker/blocker.go | 12 +- pkg/bmt/benchmark_test.go | 4 +- pkg/bmt/bmt_test.go | 8 +- pkg/bmt/proof_test.go | 2 +- pkg/bzz/utilities_test.go | 2 +- pkg/encryption/encryption.go | 2 +- pkg/encryption/mock/mock.go | 2 +- pkg/feeds/epochs/lookup_benchmark_test.go | 2 +- pkg/feeds/sequence/lookup_benchmark_test.go | 2 +- pkg/feeds/testing/lookup.go | 2 +- pkg/file/buffer_test.go | 4 +- pkg/file/joiner/joiner.go | 14 +- pkg/file/joiner/joiner_test.go | 4 +- pkg/file/joiner/redecoder_test.go | 10 +- pkg/file/pipeline/hashtrie/hashtrie_test.go | 4 +- pkg/file/redundancy/getter/getter.go | 4 +- pkg/file/redundancy/getter/getter_test.go | 6 +- pkg/file/span.go | 2 +- pkg/file/splitter/internal/job_test.go | 5 +- pkg/gsoc/gsoc.go | 2 +- pkg/gsoc/gsoc_test.go | 2 +- pkg/hive/hive.go | 18 +-- pkg/hive/hive_test.go | 8 +- pkg/jsonhttp/jsonhttp.go | 96 ++++++------- pkg/jsonhttp/jsonhttp_test.go | 4 +- pkg/jsonhttp/jsonhttptest/jsonhttptest.go | 10 +- .../jsonhttptest/testing_mock_test.go | 4 +- pkg/log/formatter.go | 18 +-- pkg/log/formatter_test.go | 34 ++--- pkg/log/httpaccess/http_access.go | 2 +- pkg/log/log.go | 10 +- pkg/log/logger.go | 18 +-- pkg/log/logger_test.go | 28 ++-- pkg/log/noop_logger.go | 20 +-- pkg/log/registry.go | 4 +- pkg/log/registry_test.go | 8 +- pkg/manifest/mantaray/marshal.go | 2 +- pkg/manifest/mantaray/marshal_test.go | 2 +- pkg/manifest/mantaray/node_test.go | 4 +- pkg/manifest/mantaray/persist_test.go | 4 +- pkg/manifest/mantaray/walker_test.go | 2 +- pkg/metrics/metrics.go | 2 +- pkg/node/bootstrap.go | 4 +- pkg/p2p/libp2p/connections_test.go | 10 +- pkg/p2p/libp2p/libp2p.go | 4 +- pkg/p2p/protobuf/protobuf_test.go | 4 +- pkg/p2p/streamtest/streamtest.go | 5 +- pkg/p2p/streamtest/streamtest_test.go | 2 +- pkg/postage/batchstore/mock/store_test.go | 6 +- pkg/postage/batchstore/store_test.go | 4 +- pkg/postage/listener/listener_test.go | 6 +- pkg/postage/service_test.go | 2 +- pkg/postage/stamper_test.go | 4 +- pkg/postage/stampissuer_test.go | 10 +- pkg/pss/mining_test.go | 2 +- pkg/pss/pss.go | 2 +- pkg/pss/pss_test.go | 2 +- pkg/pss/trojan.go | 2 +- pkg/puller/puller.go | 2 +- pkg/pullsync/pullsync_test.go | 2 +- pkg/replicas/getter.go | 12 +- pkg/replicas/putter.go | 6 +- pkg/salud/salud.go | 6 +- pkg/sharky/recovery.go | 2 +- pkg/sharky/recovery_test.go | 2 +- pkg/sharky/shard.go | 6 +- pkg/sharky/shard_slots_test.go | 12 +- pkg/sharky/sharky_test.go | 6 +- pkg/sharky/slots.go | 8 +- pkg/sharky/store.go | 12 +- pkg/shed/example_store_test.go | 2 +- pkg/shed/field_struct.go | 6 +- pkg/statestore/leveldb/leveldb.go | 4 +- pkg/statestore/mock/store.go | 4 +- pkg/statestore/storeadapter/storeadapter.go | 8 +- pkg/statestore/test/store.go | 2 +- pkg/storage/inmemstore/inmemstore.go | 8 +- pkg/storage/migration/index_test.go | 2 +- pkg/storage/migration/steps_chain_test.go | 2 +- pkg/storage/statestore.go | 4 +- pkg/storage/storagetest/benchmark.go | 8 +- pkg/storage/storagetest/storage.go | 2 +- pkg/storage/testing/chunk.go | 2 +- pkg/storageincentives/events_test.go | 4 +- pkg/storageincentives/proof_test.go | 2 +- .../redistributionstate_test.go | 6 +- pkg/storageincentives/soc_mine_test.go | 6 +- pkg/storer/compact.go | 2 +- pkg/storer/compact_test.go | 18 +-- pkg/storer/debug_test.go | 2 +- pkg/storer/export_test.go | 4 +- pkg/storer/internal/cache/cache_test.go | 4 +- pkg/storer/internal/reserve/reserve.go | 4 +- pkg/storer/internal/reserve/reserve_test.go | 26 ++-- pkg/storer/migration/refCntSize_test.go | 4 +- pkg/storer/migration/reserveRepair.go | 2 +- pkg/storer/migration/reserveRepair_test.go | 8 +- pkg/storer/migration/step_02_test.go | 4 +- pkg/storer/migration/step_04_test.go | 2 +- pkg/storer/reserve.go | 8 +- pkg/storer/reserve_test.go | 22 +-- pkg/storer/sample_test.go | 6 +- pkg/storer/subscribe_push.go | 6 +- pkg/storer/uploadstore_test.go | 4 +- pkg/storer/validate.go | 16 +-- pkg/swarm/proximity.go | 4 +- pkg/swarm/test_helpers.go | 2 +- pkg/swarm/test_helpers_test.go | 6 +- pkg/swarm/utilities_test.go | 2 +- pkg/topology/kademlia/binprefix.go | 2 +- .../kademlia/internal/metrics/metrics.go | 10 +- pkg/topology/kademlia/kademlia.go | 22 ++- pkg/topology/kademlia/kademlia_test.go | 130 +++++++++--------- pkg/topology/pslice/pslice_test.go | 12 +- pkg/tracing/tracing_test.go | 4 +- pkg/transaction/event.go | 4 +- pkg/transaction/mock/transaction.go | 8 +- pkg/transaction/transaction.go | 11 +- pkg/transaction/transaction_test.go | 6 +- pkg/util/testutil/pseudorand/reader_test.go | 2 +- 133 files changed, 498 insertions(+), 570 deletions(-) diff --git a/cmd/bee/cmd/db_test.go b/cmd/bee/cmd/db_test.go index f52da4bd0d2..7d32b5c3080 100644 --- a/cmd/bee/cmd/db_test.go +++ b/cmd/bee/cmd/db_test.go @@ -39,7 +39,7 @@ func TestDBExportImport(t *testing.T) { chunks := make(map[string]int) nChunks := 10 - for i := 0; i < nChunks; i++ { + for range nChunks { ch := storagetest.GenerateTestRandomChunk() err := db1.ReservePutter().Put(ctx, ch) if err != nil { @@ -101,13 +101,13 @@ func TestDBExportImportPinning(t *testing.T) { pins := make(map[string]any) nChunks := 10 - for i := 0; i < 2; i++ { + for range 2 { rootAddr := swarm.RandAddress(t) collection, err := db1.NewCollection(ctx) if err != nil { t.Fatal(err) } - for j := 0; j < nChunks; j++ { + for range nChunks { ch := storagetest.GenerateTestRandomChunk() err = collection.Put(ctx, ch) if err != nil { @@ -186,7 +186,7 @@ func TestDBNuke_FLAKY(t *testing.T) { }, dataDir) nChunks := 10 - for i := 0; i < nChunks; i++ { + for range nChunks { ch := storagetest.GenerateTestRandomChunk() err := db.ReservePutter().Put(ctx, ch) if err != nil { @@ -241,7 +241,7 @@ func TestDBInfo(t *testing.T) { }, dir1) nChunks := 10 - for i := 0; i < nChunks; i++ { + for range nChunks { ch := storagetest.GenerateTestRandomChunk() err := db1.ReservePutter().Put(ctx, ch) if err != nil { diff --git a/pkg/accounting/accounting.go b/pkg/accounting/accounting.go index 45e4ab14f79..a877a70d0ec 100644 --- a/pkg/accounting/accounting.go +++ b/pkg/accounting/accounting.go @@ -312,10 +312,7 @@ func (a *Accounting) PrepareCredit(ctx context.Context, peer swarm.Address, pric } } - timeElapsedInSeconds := (a.timeNow().UnixMilli() - accountingPeer.refreshTimestampMilliseconds) / 1000 - if timeElapsedInSeconds > 1 { - timeElapsedInSeconds = 1 - } + timeElapsedInSeconds := min((a.timeNow().UnixMilli()-accountingPeer.refreshTimestampMilliseconds)/1000, 1) refreshDue := new(big.Int).Mul(big.NewInt(timeElapsedInSeconds), a.refreshRate) overdraftLimit := new(big.Int).Add(accountingPeer.paymentThreshold, refreshDue) @@ -745,10 +742,7 @@ func (a *Accounting) PeerAccounting() (map[string]PeerInfo, error) { t := a.timeNow() - timeElapsedInSeconds := t.Unix() - accountingPeer.refreshReceivedTimestamp - if timeElapsedInSeconds > 1 { - timeElapsedInSeconds = 1 - } + timeElapsedInSeconds := min(t.Unix()-accountingPeer.refreshReceivedTimestamp, 1) // get appropriate refresh rate refreshRate := new(big.Int).Set(a.refreshRate) @@ -759,10 +753,7 @@ func (a *Accounting) PeerAccounting() (map[string]PeerInfo, error) { refreshDue := new(big.Int).Mul(big.NewInt(timeElapsedInSeconds), refreshRate) currentThresholdGiven := new(big.Int).Add(accountingPeer.disconnectLimit, refreshDue) - timeElapsedInSeconds = (t.UnixMilli() - accountingPeer.refreshTimestampMilliseconds) / 1000 - if timeElapsedInSeconds > 1 { - timeElapsedInSeconds = 1 - } + timeElapsedInSeconds = min((t.UnixMilli()-accountingPeer.refreshTimestampMilliseconds)/1000, 1) // get appropriate refresh rate refreshDue = new(big.Int).Mul(big.NewInt(timeElapsedInSeconds), a.refreshRate) @@ -1352,10 +1343,7 @@ func (d *debitAction) Apply() error { a.metrics.TotalDebitedAmount.Add(tot) a.metrics.DebitEventsCount.Inc() - timeElapsedInSeconds := a.timeNow().Unix() - d.accountingPeer.refreshReceivedTimestamp - if timeElapsedInSeconds > 1 { - timeElapsedInSeconds = 1 - } + timeElapsedInSeconds := min(a.timeNow().Unix()-d.accountingPeer.refreshReceivedTimestamp, 1) // get appropriate refresh rate refreshRate := new(big.Int).Set(a.refreshRate) diff --git a/pkg/accounting/accounting_test.go b/pkg/accounting/accounting_test.go index bfcd0fcda05..ec72235534e 100644 --- a/pkg/accounting/accounting_test.go +++ b/pkg/accounting/accounting_test.go @@ -1503,7 +1503,7 @@ func TestAccountingCallPaymentErrorRetries(t *testing.T) { acc.NotifyPaymentSent(peer1Addr, sentAmount, errors.New("error")) // try another n requests 1 per second - for i := 0; i < 10; i++ { + for range 10 { ts++ acc.SetTime(ts) @@ -1857,8 +1857,8 @@ func testAccountingSettlementGrowingThresholds(t *testing.T, settleFunc func(t * checkPaymentThreshold := new(big.Int).Set(testPayThreshold) // Simulate first 18 threshold upgrades - for j := 0; j < 18; j++ { - for i := 0; i < 100; i++ { + for range 18 { + for range 100 { // expect no change in threshold while less than 100 seconds worth of refreshment rate was settled settleFunc(t, acc, peer1Addr, testGrowth-1) @@ -1891,7 +1891,7 @@ func testAccountingSettlementGrowingThresholds(t *testing.T, settleFunc func(t * // Expect no increase for the next 179 seconds of refreshment - for k := 0; k < 1799; k++ { + for range 1799 { settleFunc(t, acc, peer1Addr, testGrowth) @@ -1917,7 +1917,7 @@ func testAccountingSettlementGrowingThresholds(t *testing.T, settleFunc func(t * // Expect no increase for another 3599 seconds of refreshments - for k := 0; k < 3599; k++ { + for range 3599 { settleFunc(t, acc, peer1Addr, testGrowth) diff --git a/pkg/api/api.go b/pkg/api/api.go index 418744ad621..5f747237bc4 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -624,7 +624,7 @@ func (s *Service) checkOrigin(r *http.Request) bool { // validationError is a custom error type for validation errors. type validationError struct { Entry string - Value interface{} + Value any Cause error } @@ -636,7 +636,7 @@ func (e *validationError) Error() string { // mapStructure maps the input into output struct and validates the output. // It's a helper method for the handlers, which reduces the chattiness // of the code. -func (s *Service) mapStructure(input, output interface{}) func(string, log.Logger, http.ResponseWriter) { +func (s *Service) mapStructure(input, output any) func(string, log.Logger, http.ResponseWriter) { // response unifies the response format for parsing and validation errors. response := func(err error) func(string, log.Logger, http.ResponseWriter) { return func(msg string, logger log.Logger, w http.ResponseWriter) { diff --git a/pkg/api/bzz_test.go b/pkg/api/bzz_test.go index 9738cb50f17..7f636476d1f 100644 --- a/pkg/api/bzz_test.go +++ b/pkg/api/bzz_test.go @@ -104,10 +104,7 @@ func TestBzzUploadDownloadWithRedundancy_FLAKY(t *testing.T) { store.Record() defer store.Unrecord() // we intend to forget as many chunks as possible for the given redundancy level - forget := parityCnt - if parityCnt > shardCnt { - forget = shardCnt - } + forget := min(parityCnt, shardCnt) if levels == 1 { forget = 2 } @@ -141,7 +138,7 @@ func TestBzzUploadDownloadWithRedundancy_FLAKY(t *testing.T) { if len(got) != len(want) { t.Fatalf("got %v parts, want %v parts", len(got), len(want)) } - for i := 0; i < len(want); i++ { + for i := range want { if !bytes.Equal(got[i], want[i]) { t.Errorf("part %v: got %q, want %q", i, string(got[i]), string(want[i])) } @@ -670,7 +667,7 @@ func TestBzzFilesRangeRequests(t *testing.T) { if len(got) != len(want) { t.Fatalf("got %v parts, want %v parts", len(got), len(want)) } - for i := 0; i < len(want); i++ { + for i := range want { if !bytes.Equal(got[i], want[i]) { t.Errorf("part %v: got %q, want %q", i, string(got[i]), string(want[i])) } @@ -681,7 +678,7 @@ func TestBzzFilesRangeRequests(t *testing.T) { } } -func createRangeHeader(data interface{}, ranges [][2]int) (header string, parts [][]byte) { +func createRangeHeader(data any, ranges [][2]int) (header string, parts [][]byte) { getLen := func() int { switch data := data.(type) { case []byte: diff --git a/pkg/api/chunk_stream_test.go b/pkg/api/chunk_stream_test.go index 47c8e860b51..5ba62cdd76c 100644 --- a/pkg/api/chunk_stream_test.go +++ b/pkg/api/chunk_stream_test.go @@ -38,7 +38,7 @@ func TestChunkUploadStream(t *testing.T) { t.Run("upload and verify", func(t *testing.T) { chsToGet := []swarm.Chunk{} - for i := 0; i < 5; i++ { + for range 5 { ch := testingc.GenerateTestRandomChunk() err := wsConn.SetWriteDeadline(time.Now().Add(time.Second)) diff --git a/pkg/api/export_test.go b/pkg/api/export_test.go index 1e5e8f1554f..00494f928da 100644 --- a/pkg/api/export_test.go +++ b/pkg/api/export_test.go @@ -133,7 +133,7 @@ var ErrHexLength = errHexLength type HexInvalidByteError = hexInvalidByteError -func MapStructure(input, output interface{}, hooks map[string]func(v string) (string, error)) error { +func MapStructure(input, output any, hooks map[string]func(v string) (string, error)) error { return mapStructure(input, output, hooks) } diff --git a/pkg/api/logger_test.go b/pkg/api/logger_test.go index 1e13917535f..78caf7078f3 100644 --- a/pkg/api/logger_test.go +++ b/pkg/api/logger_test.go @@ -65,8 +65,8 @@ func TestGetLoggers(t *testing.T) { } api.ReplaceLogRegistryIterateFn(fn) - have := make(map[string]interface{}) - want := make(map[string]interface{}) + have := make(map[string]any) + want := make(map[string]any) data := `{"loggers":[{"id":"b25lWzBdW10-PjgyNDYzNDg2MDM2MA==","logger":"one","subsystem":"one[0][]\u003e\u003e824634860360","verbosity":"all"},{"id":"b25lL25hbWVbMF1bXT4-ODI0NjM0ODYwMzYw","logger":"one/name","subsystem":"one/name[0][]\u003e\u003e824634860360","verbosity":"warning"},{"id":"b25lL25hbWVbMF1bXCJ2YWxcIj0xXT4-ODI0NjM0ODYwMzYw","logger":"one/name","subsystem":"one/name[0][\\\"val\\\"=1]\u003e\u003e824634860360","verbosity":"warning"},{"id":"b25lL25hbWVbMV1bXT4-ODI0NjM0ODYwMzYw","logger":"one/name","subsystem":"one/name[1][]\u003e\u003e824634860360","verbosity":"info"},{"id":"b25lL25hbWVbMl1bXT4-ODI0NjM0ODYwMzYw","logger":"one/name","subsystem":"one/name[2][]\u003e\u003e824634860360","verbosity":"info"}],"tree":{"one":{"+":["all|one[0][]\u003e\u003e824634860360"],"/":{"name":{"+":["warning|one/name[0][]\u003e\u003e824634860360","warning|one/name[0][\\\"val\\\"=1]\u003e\u003e824634860360","info|one/name[1][]\u003e\u003e824634860360","info|one/name[2][]\u003e\u003e824634860360"]}}}}}` if err := json.Unmarshal([]byte(data), &want); err != nil { t.Fatalf("unexpected error: %v", err) diff --git a/pkg/api/postage_test.go b/pkg/api/postage_test.go index bc476be99da..53ac6ccb8ae 100644 --- a/pkg/api/postage_test.go +++ b/pkg/api/postage_test.go @@ -702,7 +702,7 @@ func TestPostageAccessHandler(t *testing.T) { method string url string respCode int - resp interface{} + resp any } success := []operation{ diff --git a/pkg/api/tag.go b/pkg/api/tag.go index 6a38fd7a65b..67ebf7b7acf 100644 --- a/pkg/api/tag.go +++ b/pkg/api/tag.go @@ -19,7 +19,7 @@ import ( ) type tagRequest struct { - Address swarm.Address `json:"address,omitempty"` + Address swarm.Address `json:"address"` } type tagResponse struct { diff --git a/pkg/api/util.go b/pkg/api/util.go index 340a6270b5f..60780c5acb4 100644 --- a/pkg/api/util.go +++ b/pkg/api/util.go @@ -132,7 +132,7 @@ var flattenErrorsFormat = func(es []error) string { // // In case of parsing error, a new parseError is returned to the caller. // The caller can use the Unwrap method to get the original error. -func mapStructure(input, output interface{}, hooks map[string]func(v string) (string, error)) (err error) { +func mapStructure(input, output any, hooks map[string]func(v string) (string, error)) (err error) { if input == nil || output == nil { return nil } diff --git a/pkg/api/util_test.go b/pkg/api/util_test.go index 4628ce31270..d9ae6732810 100644 --- a/pkg/api/util_test.go +++ b/pkg/api/util_test.go @@ -102,8 +102,8 @@ func TestMapStructure(t *testing.T) { tests := []struct { name string - src interface{} - want interface{} + src any + want any wantErr error }{{ name: "bool zero value", @@ -520,7 +520,7 @@ func TestMapStructure_InputOutputSanityCheck(t *testing.T) { t.Run("input is nil", func(t *testing.T) { t.Parallel() - var input interface{} + var input any err := api.MapStructure(input, struct{}{}, nil) if err != nil { t.Fatalf("unexpected error: %v", err) @@ -541,7 +541,7 @@ func TestMapStructure_InputOutputSanityCheck(t *testing.T) { t.Parallel() var ( - input = map[string]interface{}{"someVal": "123"} + input = map[string]any{"someVal": "123"} output struct { SomeVal string `map:"someVal"` } @@ -556,8 +556,8 @@ func TestMapStructure_InputOutputSanityCheck(t *testing.T) { t.Parallel() var ( - input = map[string]interface{}{"someVal": "123"} - output interface{} + input = map[string]any{"someVal": "123"} + output any ) err := api.MapStructure(&input, output, nil) if err != nil { @@ -569,7 +569,7 @@ func TestMapStructure_InputOutputSanityCheck(t *testing.T) { t.Parallel() var ( - input = map[string]interface{}{"someVal": "123"} + input = map[string]any{"someVal": "123"} output = struct { SomeVal string `map:"someVal"` }{} @@ -584,7 +584,7 @@ func TestMapStructure_InputOutputSanityCheck(t *testing.T) { t.Parallel() var ( - input = map[string]interface{}{"someVal": "123"} + input = map[string]any{"someVal": "123"} output = "foo" ) err := api.MapStructure(&input, &output, nil) diff --git a/pkg/bitvector/bitvector_test.go b/pkg/bitvector/bitvector_test.go index d986c5624d8..c2a74e4c67e 100644 --- a/pkg/bitvector/bitvector_test.go +++ b/pkg/bitvector/bitvector_test.go @@ -64,7 +64,7 @@ func TestBitvectorGetSet(t *testing.T) { t.Errorf("error for length %v: %v", length, err) } - for i := 0; i < length; i++ { + for i := range length { if bv.Get(i) { t.Errorf("expected false for element on index %v", i) } @@ -79,9 +79,9 @@ func TestBitvectorGetSet(t *testing.T) { bv.Get(length + 8) }() - for i := 0; i < length; i++ { + for i := range length { bv.Set(i) - for j := 0; j < length; j++ { + for j := range length { if j == i { if !bv.Get(j) { t.Errorf("element on index %v is not set to true", i) diff --git a/pkg/blocker/blocker.go b/pkg/blocker/blocker.go index 5b916c81732..2d3deb10bf1 100644 --- a/pkg/blocker/blocker.go +++ b/pkg/blocker/blocker.go @@ -61,9 +61,7 @@ func New(blocklister p2p.Blocklister, flagTimeout, blockDuration, wakeUpTime tim blocklistCallback: callback, } - b.closeWg.Add(1) - go func() { - defer b.closeWg.Done() + b.closeWg.Go(func() { for { select { case <-b.quit: @@ -74,11 +72,9 @@ func New(blocklister p2p.Blocklister, flagTimeout, blockDuration, wakeUpTime tim } } } - }() + }) - b.closeWg.Add(1) - go func() { - defer b.closeWg.Done() + b.closeWg.Go(func() { for { select { case <-time.After(wakeUpTime): @@ -87,7 +83,7 @@ func New(blocklister p2p.Blocklister, flagTimeout, blockDuration, wakeUpTime tim return } } - }() + }) return b } diff --git a/pkg/bmt/benchmark_test.go b/pkg/bmt/benchmark_test.go index 7ab2339c02d..aa6cc1703dd 100644 --- a/pkg/bmt/benchmark_test.go +++ b/pkg/bmt/benchmark_test.go @@ -69,7 +69,7 @@ func benchmarkBMTBaseline(b *testing.B, _ int) { for b.Loop() { eg := new(errgroup.Group) - for j := 0; j < testSegmentCount; j++ { + for range testSegmentCount { eg.Go(func() error { _, err := bmt.Sha3hash(testData[:hashSize]) return err @@ -113,7 +113,7 @@ func benchmarkPool(b *testing.B, poolsize int) { for b.Loop() { eg := new(errgroup.Group) - for j := 0; j < cycles; j++ { + for range cycles { eg.Go(func() error { h := pool.Get() defer pool.Put(h) diff --git a/pkg/bmt/bmt_test.go b/pkg/bmt/bmt_test.go index 0eefe9fdb5d..c82fb896a6f 100644 --- a/pkg/bmt/bmt_test.go +++ b/pkg/bmt/bmt_test.go @@ -129,7 +129,7 @@ func testHasherReuse(t *testing.T, poolsize int) { h := pool.Get() defer pool.Put(h) - for i := 0; i < 100; i++ { + for i := range 100 { seed := int64(i) testData := testutil.RandBytesWithSeed(t, 4096, seed) n := rand.Intn(h.Capacity()) @@ -151,7 +151,7 @@ func TestBMTConcurrentUse(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() eg, ectx := errgroup.WithContext(ctx) - for i := 0; i < cycles; i++ { + for range cycles { eg.Go(func() error { select { case <-ectx.Done(): @@ -204,7 +204,7 @@ func TestBMTWriterBuffers(t *testing.T) { reads := rand.Intn(count*2-1) + 1 offsets := make([]int, reads+1) - for i := 0; i < reads; i++ { + for i := range reads { offsets[i] = rand.Intn(size) + 1 } offsets[reads] = size @@ -235,7 +235,7 @@ func TestBMTWriterBuffers(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() eg, ectx := errgroup.WithContext(ctx) - for i := 0; i < attempts; i++ { + for range attempts { eg.Go(func() error { select { case <-ectx.Done(): diff --git a/pkg/bmt/proof_test.go b/pkg/bmt/proof_test.go index ba1c3e7220c..5b3625acefb 100644 --- a/pkg/bmt/proof_test.go +++ b/pkg/bmt/proof_test.go @@ -209,7 +209,7 @@ func TestProof(t *testing.T) { t.Fatal(err) } - for i := 0; i < 128; i++ { + for i := range 128 { t.Run(fmt.Sprintf("segmentIndex %d", i), func(t *testing.T) { t.Parallel() diff --git a/pkg/bzz/utilities_test.go b/pkg/bzz/utilities_test.go index 7e4666b66ac..e92e7e11e13 100644 --- a/pkg/bzz/utilities_test.go +++ b/pkg/bzz/utilities_test.go @@ -45,7 +45,7 @@ func makeAddreses(t *testing.T, count int) []bzz.Address { t.Helper() result := make([]bzz.Address, count) - for i := 0; i < count; i++ { + for i := range count { result[i] = makeAddress(t) } return result diff --git a/pkg/encryption/encryption.go b/pkg/encryption/encryption.go index 97a582814c8..d46d567088d 100644 --- a/pkg/encryption/encryption.go +++ b/pkg/encryption/encryption.go @@ -158,7 +158,7 @@ func (e *Encryption) Transcrypt(i int, in, out []byte) error { // XOR bytes uptil length of in (out must be at least as long) inLength := len(in) - for j := 0; j < inLength; j++ { + for j := range inLength { out[j] = in[j] ^ segmentKey[j] } // insert padding if out is longer diff --git a/pkg/encryption/mock/mock.go b/pkg/encryption/mock/mock.go index 66420fb2f1b..b917b095be7 100644 --- a/pkg/encryption/mock/mock.go +++ b/pkg/encryption/mock/mock.go @@ -124,7 +124,7 @@ func xor(input, key []byte) ([]byte, error) { } inputLen := len(input) output := make([]byte, inputLen) - for i := 0; i < inputLen; i++ { + for i := range inputLen { output[i] = input[i] ^ key[i%keyLen] } return output, nil diff --git a/pkg/feeds/epochs/lookup_benchmark_test.go b/pkg/feeds/epochs/lookup_benchmark_test.go index 9d1005ab023..0d409bef686 100644 --- a/pkg/feeds/epochs/lookup_benchmark_test.go +++ b/pkg/feeds/epochs/lookup_benchmark_test.go @@ -38,7 +38,7 @@ func BenchmarkFinder(b *testing.B) { ctx := context.Background() - for at := int64(0); at < prefill; at++ { + for at := range prefill { err = updater.Update(ctx, at, payload) if err != nil { b.Fatal(err) diff --git a/pkg/feeds/sequence/lookup_benchmark_test.go b/pkg/feeds/sequence/lookup_benchmark_test.go index d10fe144aca..73e9255d497 100644 --- a/pkg/feeds/sequence/lookup_benchmark_test.go +++ b/pkg/feeds/sequence/lookup_benchmark_test.go @@ -36,7 +36,7 @@ func BenchmarkFinder(b *testing.B) { ctx := context.Background() - for at := int64(0); at < prefill; at++ { + for at := range prefill { err = updater.Update(ctx, at, payload) if err != nil { b.Fatal(err) diff --git a/pkg/feeds/testing/lookup.go b/pkg/feeds/testing/lookup.go index 21656be5b64..e56e3c477ba 100644 --- a/pkg/feeds/testing/lookup.go +++ b/pkg/feeds/testing/lookup.go @@ -186,7 +186,7 @@ func TestFinderIntervals(t *testing.T, nextf func() (bool, int64), finderf func( func TestFinderRandomIntervals(t *testing.T, finderf func(storage.Getter, *feeds.Feed) feeds.Lookup, updaterf func(putter storage.Putter, signer crypto.Signer, topic []byte) (feeds.Updater, error)) { t.Parallel() - for j := 0; j < 3; j++ { + for j := range 3 { t.Run(fmt.Sprintf("random intervals %d", j), func(t *testing.T) { t.Parallel() diff --git a/pkg/file/buffer_test.go b/pkg/file/buffer_test.go index 9fdf66266b6..71b374b7f4b 100644 --- a/pkg/file/buffer_test.go +++ b/pkg/file/buffer_test.go @@ -119,8 +119,8 @@ func TestCopyBuffer(t *testing.T) { dataSize int }{} - for i := 0; i < len(readBufferSizes); i++ { - for j := 0; j < len(dataSizes); j++ { + for i := range readBufferSizes { + for j := range dataSizes { testCases = append(testCases, struct { readBufferSize int dataSize int diff --git a/pkg/file/joiner/joiner.go b/pkg/file/joiner/joiner.go index 313ea21bd2c..cc89f921397 100644 --- a/pkg/file/joiner/joiner.go +++ b/pkg/file/joiner/joiner.go @@ -209,10 +209,7 @@ func (j *joiner) ReadAt(buffer []byte, off int64) (read int, err error) { return 0, io.EOF } - readLen := int64(cap(buffer)) - if readLen > j.span-off { - readLen = j.span - off - } + readLen := min(int64(cap(buffer)), j.span-off) var bytesRead int64 var eg errgroup.Group j.readAtOffset(buffer, j.rootData, 0, j.span, off, 0, readLen, &bytesRead, j.rootParity, &eg) @@ -277,14 +274,9 @@ func (j *joiner) readAtOffset( subtrieSpanLimit := sec currentReadSize := subtrieSpan - (off - cur) // the size of the subtrie, minus the offset from the start of the trie - // upper bound alignments - if currentReadSize > bytesToRead { - currentReadSize = bytesToRead - } - if currentReadSize > subtrieSpan { - currentReadSize = subtrieSpan - } + currentReadSize = min(currentReadSize, bytesToRead) + currentReadSize = min(currentReadSize, subtrieSpan) func(address swarm.Address, b []byte, cur, subTrieSize, off, bufferOffset, bytesToRead, subtrieSpanLimit int64) { eg.Go(func() error { diff --git a/pkg/file/joiner/joiner_test.go b/pkg/file/joiner/joiner_test.go index 27591252548..a8087a859db 100644 --- a/pkg/file/joiner/joiner_test.go +++ b/pkg/file/joiner/joiner_test.go @@ -790,7 +790,7 @@ func TestJoinerTwoLevelsAcrossChunk(t *testing.T) { // create 128+1 chunks for all references in the intermediate chunks cursor := 8 - for i := 0; i < swarm.Branches; i++ { + for range swarm.Branches { chunkAddressBytes := firstChunk.Data()[cursor : cursor+swarm.SectionSize] chunkAddress := swarm.NewAddress(chunkAddressBytes) ch := filetest.GenerateTestRandomFileChunk(chunkAddress, swarm.ChunkSize, swarm.ChunkSize) @@ -815,7 +815,7 @@ func TestJoinerTwoLevelsAcrossChunk(t *testing.T) { // read back all the chunks and verify b := make([]byte, swarm.ChunkSize) - for i := 0; i < swarm.Branches; i++ { + for i := range swarm.Branches { c, err := j.Read(b) if err != nil { t.Fatal(err) diff --git a/pkg/file/joiner/redecoder_test.go b/pkg/file/joiner/redecoder_test.go index 7df041de76d..0cf3b604ec2 100644 --- a/pkg/file/joiner/redecoder_test.go +++ b/pkg/file/joiner/redecoder_test.go @@ -33,7 +33,7 @@ func TestReDecoderFlow(t *testing.T) { // Create real data chunks with proper content dataShards := make([][]byte, dataShardCount) - for i := 0; i < dataShardCount; i++ { + for i := range dataShardCount { // Create chunks with simpler test data dataShards[i] = make([]byte, swarm.ChunkWithSpanSize) // Create a unique string for this shard @@ -44,7 +44,7 @@ func TestReDecoderFlow(t *testing.T) { // Create parity chunks using Reed-Solomon encoding parityShards := make([][]byte, parityShardCount) - for i := 0; i < parityShardCount; i++ { + for i := range parityShardCount { parityShards[i] = make([]byte, swarm.ChunkWithSpanSize) } @@ -68,7 +68,7 @@ func TestReDecoderFlow(t *testing.T) { addresses := make([]swarm.Address, totalShardCount) chunks := make([]swarm.Chunk, totalShardCount) - for i := 0; i < totalShardCount; i++ { + for i := range totalShardCount { // Create proper content-addressed chunks chunk, err := cac.NewWithDataSpan(allShards[i]) if err != nil { @@ -125,7 +125,7 @@ func TestReDecoderFlow(t *testing.T) { // we can still access the chunks // Sanity check - verify we can still fetch chunks through the cache - for i := 0; i < dataShardCount; i++ { + for i := range dataShardCount { _, err := decoder.Get(ctx, addresses[i]) if err != nil { t.Fatalf("Failed to get chunk %d after recovery: %v", i, err) @@ -139,7 +139,7 @@ func TestReDecoderFlow(t *testing.T) { } // Verify all chunks can be fetched through the ReDecoder - for i := 0; i < dataShardCount; i++ { + for i := range dataShardCount { _, err := newDecoder.Get(ctx, addresses[i]) if err != nil { t.Fatalf("Failed to get chunk %d through ReDecoder: %v", i, err) diff --git a/pkg/file/pipeline/hashtrie/hashtrie_test.go b/pkg/file/pipeline/hashtrie/hashtrie_test.go index edbe3c9e2e5..9f502f3fd5d 100644 --- a/pkg/file/pipeline/hashtrie/hashtrie_test.go +++ b/pkg/file/pipeline/hashtrie/hashtrie_test.go @@ -198,7 +198,7 @@ func TestLevels_TrieFull(t *testing.T) { ) // to create a level wrap we need to do branching^(level-1) writes - for i := 0; i < writes; i++ { + for range writes { a := &pipeline.PipeWriteArgs{Ref: addr.Bytes(), Span: span} err := ht.ChainWrite(a) if err != nil { @@ -239,7 +239,7 @@ func TestRegression(t *testing.T) { ) binary.LittleEndian.PutUint64(span, 4096) - for i := 0; i < writes; i++ { + for range writes { a := &pipeline.PipeWriteArgs{Ref: addr.Bytes(), Span: span} err := ht.ChainWrite(a) if err != nil { diff --git a/pkg/file/redundancy/getter/getter.go b/pkg/file/redundancy/getter/getter.go index 0953c501dc5..30ec23c00b3 100644 --- a/pkg/file/redundancy/getter/getter.go +++ b/pkg/file/redundancy/getter/getter.go @@ -69,12 +69,12 @@ func New(addrs []swarm.Address, shardCnt int, g storage.Getter, p storage.Putter } // after init, cache and wait channels are immutable, need no locking - for i := 0; i < shardCnt; i++ { + for i := range shardCnt { d.cache[addrs[i].ByteString()] = i } // after init, cache and wait channels are immutable, need no locking - for i := 0; i < size; i++ { + for i := range size { d.waits[i] = make(chan error) } diff --git a/pkg/file/redundancy/getter/getter_test.go b/pkg/file/redundancy/getter/getter_test.go index 42566866a3e..f3c9a27c3ff 100644 --- a/pkg/file/redundancy/getter/getter_test.go +++ b/pkg/file/redundancy/getter/getter_test.go @@ -266,7 +266,7 @@ func initData(t *testing.T, buf [][]byte, shardCnt int, s storage.ChunkStore) [] spanBytes := make([]byte, 8) binary.LittleEndian.PutUint64(spanBytes, swarm.ChunkSize) - for i := 0; i < len(buf); i++ { + for i := range buf { buf[i] = make([]byte, swarm.ChunkWithSpanSize) if i >= shardCnt { continue @@ -291,7 +291,7 @@ func initData(t *testing.T, buf [][]byte, shardCnt int, s storage.ChunkStore) [] // calculate chunk addresses and upload to the store addrs := make([]swarm.Address, len(buf)) ctx := context.TODO() - for i := 0; i < len(buf); i++ { + for i := range buf { chunk, err := cac.NewWithDataSpan(buf[i]) if err != nil { t.Fatal(err) @@ -313,7 +313,7 @@ func checkShardsAvailable(t *testing.T, s storage.ChunkStore, addrs []swarm.Addr eg.Go(func() (err error) { var delay time.Duration var ch swarm.Chunk - for i := 0; i < 30; i++ { + for i := range 30 { select { case <-ctx.Done(): return ctx.Err() diff --git a/pkg/file/span.go b/pkg/file/span.go index b8366233bd2..681cafb4acb 100644 --- a/pkg/file/span.go +++ b/pkg/file/span.go @@ -22,7 +22,7 @@ func GenerateSpanSizes(levels, branches int) []int64 { spans := make([]int64, levels) branchesSixtyfour := int64(branches) var span int64 = 1 - for i := 0; i < 9; i++ { + for i := range 9 { spans[i] = span span *= branchesSixtyfour } diff --git a/pkg/file/splitter/internal/job_test.go b/pkg/file/splitter/internal/job_test.go index 7e9824d4748..98a44ce3f51 100644 --- a/pkg/file/splitter/internal/job_test.go +++ b/pkg/file/splitter/internal/job_test.go @@ -82,10 +82,7 @@ func testSplitterJobVector(t *testing.T) { j := internal.NewSimpleSplitterJob(ctx, store, int64(len(data)), false) for i := 0; i < len(data); i += swarm.ChunkSize { - l := swarm.ChunkSize - if len(data)-i < swarm.ChunkSize { - l = len(data) - i - } + l := min(len(data)-i, swarm.ChunkSize) c, err := j.Write(data[i : i+l]) if err != nil { t.Fatal(err) diff --git a/pkg/gsoc/gsoc.go b/pkg/gsoc/gsoc.go index 6497c9b0c63..464681258e2 100644 --- a/pkg/gsoc/gsoc.go +++ b/pkg/gsoc/gsoc.go @@ -50,7 +50,7 @@ func (l *listener) Subscribe(address swarm.Address, handler Handler) (cleanup fu defer l.handlersMu.Unlock() h := l.handlers[address.ByteString()] - for i := 0; i < len(h); i++ { + for i := range h { if h[i] == &handler { l.handlers[address.ByteString()] = append(h[:i], h[i+1:]...) return diff --git a/pkg/gsoc/gsoc_test.go b/pkg/gsoc/gsoc_test.go index dfbe8e03a5c..dc49b0809a8 100644 --- a/pkg/gsoc/gsoc_test.go +++ b/pkg/gsoc/gsoc_test.go @@ -114,7 +114,7 @@ func ensureCalls(t *testing.T, calls *int, exp int) { func waitHandlerCallback(t *testing.T, msgChan *chan struct{}, count int) { t.Helper() - for received := 0; received < count; received++ { + for range count { select { case <-*msgChan: case <-time.After(1 * time.Second): diff --git a/pkg/hive/hive.go b/pkg/hive/hive.go index 27858cdcf87..7932fffc401 100644 --- a/pkg/hive/hive.go +++ b/pkg/hive/hive.go @@ -249,31 +249,25 @@ func (s *Service) disconnect(peer p2p.Peer) error { func (s *Service) startCheckPeersHandler() { ctx, cancel := context.WithCancel(context.Background()) - s.wg.Add(1) - go func() { - defer s.wg.Done() + s.wg.Go(func() { <-s.quit cancel() - }() + }) - s.wg.Add(1) - go func() { - defer s.wg.Done() + s.wg.Go(func() { for { select { case <-ctx.Done(): return case newPeers := <-s.peersChan: - s.wg.Add(1) - go func() { - defer s.wg.Done() + s.wg.Go(func() { cctx, cancel := context.WithTimeout(ctx, batchValidationTimeout) defer cancel() s.checkAndAddPeers(cctx, newPeers) - }() + }) } } - }() + }) } func (s *Service) checkAndAddPeers(ctx context.Context, peers pb.Peers) { diff --git a/pkg/hive/hive_test.go b/pkg/hive/hive_test.go index fdb903cf7e5..df06c2674ed 100644 --- a/pkg/hive/hive_test.go +++ b/pkg/hive/hive_test.go @@ -119,11 +119,11 @@ func TestBroadcastPeers(t *testing.T) { // populate all expected and needed random resources for 2 full batches // tests cases that uses fewer resources can use sub-slices of this data - var bzzAddresses []bzz.Address - var overlays []swarm.Address - var wantMsgs []pb.Peers + bzzAddresses := make([]bzz.Address, 0, 2*hive.MaxBatchSize) + overlays := make([]swarm.Address, 0, 2*hive.MaxBatchSize) + wantMsgs := make([]pb.Peers, 0, 2*hive.MaxBatchSize) - for i := 0; i < 2; i++ { + for range 2 { wantMsgs = append(wantMsgs, pb.Peers{Peers: []*pb.BzzAddress{}}) } diff --git a/pkg/jsonhttp/jsonhttp.go b/pkg/jsonhttp/jsonhttp.go index d9806dca523..e555223e110 100644 --- a/pkg/jsonhttp/jsonhttp.go +++ b/pkg/jsonhttp/jsonhttp.go @@ -44,7 +44,7 @@ type StatusResponse struct { } // Respond writes a JSON-encoded body to http.ResponseWriter. -func Respond(w http.ResponseWriter, statusCode int, response interface{}) { +func Respond(w http.ResponseWriter, statusCode int, response any) { if statusCode == 0 { statusCode = http.StatusOK } @@ -88,32 +88,32 @@ func Respond(w http.ResponseWriter, statusCode int, response interface{}) { } // Continue writes a response with status code 100. -func Continue(w http.ResponseWriter, response interface{}) { +func Continue(w http.ResponseWriter, response any) { Respond(w, http.StatusContinue, response) } // SwitchingProtocols writes a response with status code 101. -func SwitchingProtocols(w http.ResponseWriter, response interface{}) { +func SwitchingProtocols(w http.ResponseWriter, response any) { Respond(w, http.StatusSwitchingProtocols, response) } // OK writes a response with status code 200. -func OK(w http.ResponseWriter, response interface{}) { +func OK(w http.ResponseWriter, response any) { Respond(w, http.StatusOK, response) } // Created writes a response with status code 201. -func Created(w http.ResponseWriter, response interface{}) { +func Created(w http.ResponseWriter, response any) { Respond(w, http.StatusCreated, response) } // Accepted writes a response with status code 202. -func Accepted(w http.ResponseWriter, response interface{}) { +func Accepted(w http.ResponseWriter, response any) { Respond(w, http.StatusAccepted, response) } // NonAuthoritativeInfo writes a response with status code 203. -func NonAuthoritativeInfo(w http.ResponseWriter, response interface{}) { +func NonAuthoritativeInfo(w http.ResponseWriter, response any) { Respond(w, http.StatusNonAuthoritativeInfo, response) } @@ -125,206 +125,206 @@ func NoContent(w http.ResponseWriter) { } // ResetContent writes a response with status code 205. -func ResetContent(w http.ResponseWriter, response interface{}) { +func ResetContent(w http.ResponseWriter, response any) { Respond(w, http.StatusResetContent, response) } // PartialContent writes a response with status code 206. -func PartialContent(w http.ResponseWriter, response interface{}) { +func PartialContent(w http.ResponseWriter, response any) { Respond(w, http.StatusPartialContent, response) } // MultipleChoices writes a response with status code 300. -func MultipleChoices(w http.ResponseWriter, response interface{}) { +func MultipleChoices(w http.ResponseWriter, response any) { Respond(w, http.StatusMultipleChoices, response) } // MovedPermanently writes a response with status code 301. -func MovedPermanently(w http.ResponseWriter, response interface{}) { +func MovedPermanently(w http.ResponseWriter, response any) { Respond(w, http.StatusMovedPermanently, response) } // Found writes a response with status code 302. -func Found(w http.ResponseWriter, response interface{}) { +func Found(w http.ResponseWriter, response any) { Respond(w, http.StatusFound, response) } // SeeOther writes a response with status code 303. -func SeeOther(w http.ResponseWriter, response interface{}) { +func SeeOther(w http.ResponseWriter, response any) { Respond(w, http.StatusSeeOther, response) } // NotModified writes a response with status code 304. -func NotModified(w http.ResponseWriter, response interface{}) { +func NotModified(w http.ResponseWriter, response any) { Respond(w, http.StatusNotModified, response) } // UseProxy writes a response with status code 305. -func UseProxy(w http.ResponseWriter, response interface{}) { +func UseProxy(w http.ResponseWriter, response any) { Respond(w, http.StatusUseProxy, response) } // TemporaryRedirect writes a response with status code 307. -func TemporaryRedirect(w http.ResponseWriter, response interface{}) { +func TemporaryRedirect(w http.ResponseWriter, response any) { Respond(w, http.StatusTemporaryRedirect, response) } // PermanentRedirect writes a response with status code 308. -func PermanentRedirect(w http.ResponseWriter, response interface{}) { +func PermanentRedirect(w http.ResponseWriter, response any) { Respond(w, http.StatusPermanentRedirect, response) } // BadRequest writes a response with status code 400. -func BadRequest(w http.ResponseWriter, response interface{}) { +func BadRequest(w http.ResponseWriter, response any) { Respond(w, http.StatusBadRequest, response) } // Unauthorized writes a response with status code 401. -func Unauthorized(w http.ResponseWriter, response interface{}) { +func Unauthorized(w http.ResponseWriter, response any) { Respond(w, http.StatusUnauthorized, response) } // PaymentRequired writes a response with status code 402. -func PaymentRequired(w http.ResponseWriter, response interface{}) { +func PaymentRequired(w http.ResponseWriter, response any) { Respond(w, http.StatusPaymentRequired, response) } // Forbidden writes a response with status code 403. -func Forbidden(w http.ResponseWriter, response interface{}) { +func Forbidden(w http.ResponseWriter, response any) { Respond(w, http.StatusForbidden, response) } // NotFound writes a response with status code 404. -func NotFound(w http.ResponseWriter, response interface{}) { +func NotFound(w http.ResponseWriter, response any) { Respond(w, http.StatusNotFound, response) } // MethodNotAllowed writes a response with status code 405. -func MethodNotAllowed(w http.ResponseWriter, response interface{}) { +func MethodNotAllowed(w http.ResponseWriter, response any) { Respond(w, http.StatusMethodNotAllowed, response) } // NotAcceptable writes a response with status code 406. -func NotAcceptable(w http.ResponseWriter, response interface{}) { +func NotAcceptable(w http.ResponseWriter, response any) { Respond(w, http.StatusNotAcceptable, response) } // ProxyAuthRequired writes a response with status code 407. -func ProxyAuthRequired(w http.ResponseWriter, response interface{}) { +func ProxyAuthRequired(w http.ResponseWriter, response any) { Respond(w, http.StatusProxyAuthRequired, response) } // RequestTimeout writes a response with status code 408. -func RequestTimeout(w http.ResponseWriter, response interface{}) { +func RequestTimeout(w http.ResponseWriter, response any) { Respond(w, http.StatusRequestTimeout, response) } // Conflict writes a response with status code 409. -func Conflict(w http.ResponseWriter, response interface{}) { +func Conflict(w http.ResponseWriter, response any) { Respond(w, http.StatusConflict, response) } // Gone writes a response with status code 410. -func Gone(w http.ResponseWriter, response interface{}) { +func Gone(w http.ResponseWriter, response any) { Respond(w, http.StatusGone, response) } // LengthRequired writes a response with status code 411. -func LengthRequired(w http.ResponseWriter, response interface{}) { +func LengthRequired(w http.ResponseWriter, response any) { Respond(w, http.StatusLengthRequired, response) } // PreconditionFailed writes a response with status code 412. -func PreconditionFailed(w http.ResponseWriter, response interface{}) { +func PreconditionFailed(w http.ResponseWriter, response any) { Respond(w, http.StatusPreconditionFailed, response) } // RequestEntityTooLarge writes a response with status code 413. -func RequestEntityTooLarge(w http.ResponseWriter, response interface{}) { +func RequestEntityTooLarge(w http.ResponseWriter, response any) { Respond(w, http.StatusRequestEntityTooLarge, response) } // RequestURITooLong writes a response with status code 414. -func RequestURITooLong(w http.ResponseWriter, response interface{}) { +func RequestURITooLong(w http.ResponseWriter, response any) { Respond(w, http.StatusRequestURITooLong, response) } // UnsupportedMediaType writes a response with status code 415. -func UnsupportedMediaType(w http.ResponseWriter, response interface{}) { +func UnsupportedMediaType(w http.ResponseWriter, response any) { Respond(w, http.StatusUnsupportedMediaType, response) } // RequestedRangeNotSatisfiable writes a response with status code 416. -func RequestedRangeNotSatisfiable(w http.ResponseWriter, response interface{}) { +func RequestedRangeNotSatisfiable(w http.ResponseWriter, response any) { Respond(w, http.StatusRequestedRangeNotSatisfiable, response) } // ExpectationFailed writes a response with status code 417. -func ExpectationFailed(w http.ResponseWriter, response interface{}) { +func ExpectationFailed(w http.ResponseWriter, response any) { Respond(w, http.StatusExpectationFailed, response) } // Teapot writes a response with status code 418. -func Teapot(w http.ResponseWriter, response interface{}) { +func Teapot(w http.ResponseWriter, response any) { Respond(w, http.StatusTeapot, response) } // UnprocessableEntity writes a response with status code 422. -func UnprocessableEntity(w http.ResponseWriter, response interface{}) { +func UnprocessableEntity(w http.ResponseWriter, response any) { Respond(w, http.StatusUnprocessableEntity, response) } // UpgradeRequired writes a response with status code 426. -func UpgradeRequired(w http.ResponseWriter, response interface{}) { +func UpgradeRequired(w http.ResponseWriter, response any) { Respond(w, http.StatusUpgradeRequired, response) } // PreconditionRequired writes a response with status code 428. -func PreconditionRequired(w http.ResponseWriter, response interface{}) { +func PreconditionRequired(w http.ResponseWriter, response any) { Respond(w, http.StatusPreconditionRequired, response) } // TooManyRequests writes a response with status code 429. -func TooManyRequests(w http.ResponseWriter, response interface{}) { +func TooManyRequests(w http.ResponseWriter, response any) { Respond(w, http.StatusTooManyRequests, response) } // RequestHeaderFieldsTooLarge writes a response with status code 431. -func RequestHeaderFieldsTooLarge(w http.ResponseWriter, response interface{}) { +func RequestHeaderFieldsTooLarge(w http.ResponseWriter, response any) { Respond(w, http.StatusRequestHeaderFieldsTooLarge, response) } // UnavailableForLegalReasons writes a response with status code 451. -func UnavailableForLegalReasons(w http.ResponseWriter, response interface{}) { +func UnavailableForLegalReasons(w http.ResponseWriter, response any) { Respond(w, http.StatusUnavailableForLegalReasons, response) } // InternalServerError writes a response with status code 500. -func InternalServerError(w http.ResponseWriter, response interface{}) { +func InternalServerError(w http.ResponseWriter, response any) { Respond(w, http.StatusInternalServerError, response) } // NotImplemented writes a response with status code 501. -func NotImplemented(w http.ResponseWriter, response interface{}) { +func NotImplemented(w http.ResponseWriter, response any) { Respond(w, http.StatusNotImplemented, response) } // BadGateway writes a response with status code 502. -func BadGateway(w http.ResponseWriter, response interface{}) { +func BadGateway(w http.ResponseWriter, response any) { Respond(w, http.StatusBadGateway, response) } // ServiceUnavailable writes a response with status code 503. -func ServiceUnavailable(w http.ResponseWriter, response interface{}) { +func ServiceUnavailable(w http.ResponseWriter, response any) { Respond(w, http.StatusServiceUnavailable, response) } // GatewayTimeout writes a response with status code 504. -func GatewayTimeout(w http.ResponseWriter, response interface{}) { +func GatewayTimeout(w http.ResponseWriter, response any) { Respond(w, http.StatusGatewayTimeout, response) } // HTTPVersionNotSupported writes a response with status code 505. -func HTTPVersionNotSupported(w http.ResponseWriter, response interface{}) { +func HTTPVersionNotSupported(w http.ResponseWriter, response any) { Respond(w, http.StatusHTTPVersionNotSupported, response) } diff --git a/pkg/jsonhttp/jsonhttp_test.go b/pkg/jsonhttp/jsonhttp_test.go index 61a17684e5e..43b7b90b5ad 100644 --- a/pkg/jsonhttp/jsonhttp_test.go +++ b/pkg/jsonhttp/jsonhttp_test.go @@ -134,7 +134,7 @@ func TestRespond_special(t *testing.T) { for _, tc := range []struct { name string code int - response interface{} + response any wantMessage string }{ { @@ -245,7 +245,7 @@ func TestStandardHTTPResponds(t *testing.T) { t.Parallel() for _, tc := range []struct { - f func(w http.ResponseWriter, response interface{}) + f func(w http.ResponseWriter, response any) code int }{ {f: jsonhttp.Continue, code: http.StatusContinue}, diff --git a/pkg/jsonhttp/jsonhttptest/jsonhttptest.go b/pkg/jsonhttp/jsonhttptest/jsonhttptest.go index 7902a566dc9..9e64b0a8422 100644 --- a/pkg/jsonhttp/jsonhttptest/jsonhttptest.go +++ b/pkg/jsonhttp/jsonhttptest/jsonhttptest.go @@ -157,7 +157,7 @@ func WithRequestBody(body io.Reader) Option { // WithJSONRequestBody writes a request JSON-encoded body to the request made by // the Request function. -func WithJSONRequestBody(r interface{}) Option { +func WithJSONRequestBody(r any) Option { return optionFunc(func(o *options) error { b, err := json.Marshal(r) if err != nil { @@ -256,7 +256,7 @@ func WithNonEmptyResponseHeader(key string) Option { // WithExpectedJSONResponse validates that the response from the request in the // Request function matches JSON-encoded body provided here. -func WithExpectedJSONResponse(response interface{}) Option { +func WithExpectedJSONResponse(response any) Option { return optionFunc(func(o *options) error { o.expectedJSONResponse = response return nil @@ -265,7 +265,7 @@ func WithExpectedJSONResponse(response interface{}) Option { // WithUnmarshalJSONResponse unmarshals response body from the request in the // Request function to the provided response. Response must be a pointer. -func WithUnmarshalJSONResponse(response interface{}) Option { +func WithUnmarshalJSONResponse(response any) Option { return optionFunc(func(o *options) error { o.unmarshalResponse = response return nil @@ -304,8 +304,8 @@ type options struct { expectedResponseHeaders http.Header nonEmptyResponseHeaders []string expectedResponse []byte - expectedJSONResponse interface{} - unmarshalResponse interface{} + expectedJSONResponse any + unmarshalResponse any responseBody *[]byte noResponseBody bool } diff --git a/pkg/jsonhttp/jsonhttptest/testing_mock_test.go b/pkg/jsonhttp/jsonhttptest/testing_mock_test.go index c496f13d8ae..0d375a926c3 100644 --- a/pkg/jsonhttp/jsonhttptest/testing_mock_test.go +++ b/pkg/jsonhttp/jsonhttptest/testing_mock_test.go @@ -62,11 +62,11 @@ func (m *mock) Helper() { m.isHelper = true } -func (m *mock) Errorf(format string, args ...interface{}) { +func (m *mock) Errorf(format string, args ...any) { m.got.errors = append(m.got.errors, fmt.Sprintf(format, args...)) } -func (m *mock) Fatal(args ...interface{}) { +func (m *mock) Fatal(args ...any) { m.got.fatal = fmt.Sprint(args...) panic(errFailed) // terminate the goroutine to detect it in the assert function } diff --git a/pkg/log/formatter.go b/pkg/log/formatter.go index bf0a71a9a7e..558e445376a 100644 --- a/pkg/log/formatter.go +++ b/pkg/log/formatter.go @@ -45,12 +45,12 @@ type Marshaler interface { // with exported fields // // It may return any value of any type. - MarshalLog() interface{} + MarshalLog() any } // PseudoStruct is a list of key-value pairs that gets logged as a struct. // E.g.: PseudoStruct{"f1", 1, "f2", true, "f3", []int{}}. -type PseudoStruct []interface{} +type PseudoStruct []any // fmtOptions carries parameters which influence the way logs are generated/formatted. type fmtOptions struct { @@ -97,7 +97,7 @@ type formatter struct { // render produces a log line where the base is // never escaped; the opposite is true for args. -func (f *formatter) render(base, args []interface{}) []byte { +func (f *formatter) render(base, args []any) []byte { buf := bytes.NewBuffer(make([]byte, 0, 1024)) if f.opts.jsonOutput { buf.WriteByte('{') @@ -116,7 +116,7 @@ func (f *formatter) render(base, args []interface{}) []byte { // separator (which depends on the output format) before the first pair is // written. If escapeKeys is true, the keys are assumed to have // non-JSON-compatible characters in them and must be evaluated for escapes. -func (f *formatter) flatten(buf *bytes.Buffer, kvList []interface{}, continuing bool, escapeKeys bool) { +func (f *formatter) flatten(buf *bytes.Buffer, kvList []any, continuing bool, escapeKeys bool) { // This logic overlaps with sanitize() but saves one type-cast per key, // which can be measurable. if len(kvList)%2 != 0 { @@ -159,7 +159,7 @@ func (f *formatter) flatten(buf *bytes.Buffer, kvList []interface{}, continuing // prettyWithFlags prettifies the given value. // TODO: This is not fast. Most of the overhead goes here. -func (f *formatter) prettyWithFlags(value interface{}, flags uint32, depth int) string { +func (f *formatter) prettyWithFlags(value any, flags uint32, depth int) string { const flagRawStruct = 0x1 // Do not print braces on structs. if depth > f.opts.maxLogDepth { @@ -387,12 +387,12 @@ func (f *formatter) caller() caller { } // nonStringKey converts non-string value v to string. -func (f *formatter) nonStringKey(v interface{}) string { +func (f *formatter) nonStringKey(v any) string { return fmt.Sprintf("", f.snippet(v)) } // snippet produces a short snippet string of an arbitrary value. -func (f *formatter) snippet(v interface{}) string { +func (f *formatter) snippet(v any) string { const snipLen = 16 snip := f.prettyWithFlags(v, 0, 0) @@ -405,7 +405,7 @@ func (f *formatter) snippet(v interface{}) string { // sanitize ensures that a list of key-value pairs has a value for every key // (adding a value if needed) and that each key is a string (substituting a key // if needed). -func (f *formatter) sanitize(kvList []interface{}) []interface{} { +func (f *formatter) sanitize(kvList []any) []any { if len(kvList)%2 != 0 { kvList = append(kvList, noValue) } @@ -466,7 +466,7 @@ func needsEscape(s string) bool { } // invokeMarshaler returns panic-safe output from the Marshaler.MarshalLog() method. -func invokeMarshaler(m Marshaler) (ret interface{}) { +func invokeMarshaler(m Marshaler) (ret any) { defer func() { if r := recover(); r != nil { ret = fmt.Sprintf("", r) diff --git a/pkg/log/formatter_test.go b/pkg/log/formatter_test.go index 7dd44b87104..c78cce38453 100644 --- a/pkg/log/formatter_test.go +++ b/pkg/log/formatter_test.go @@ -36,7 +36,7 @@ func (p pointErr) MarshalText() ([]byte, error) { // marshalerTest expect to result in the MarshalLog() value when logged. type marshalerTest struct{ val string } -func (marshalerTest) MarshalLog() interface{} { +func (marshalerTest) MarshalLog() any { return struct{ Inner string }{"I am a log.Marshaler"} } func (marshalerTest) String() string { @@ -50,7 +50,7 @@ func (marshalerTest) Error() string { // marshalerPanicTest expect this to result in a panic when logged. type marshalerPanicTest struct{ val string } -func (marshalerPanicTest) MarshalLog() interface{} { +func (marshalerPanicTest) MarshalLog() any { panic("marshalerPanicTest") } @@ -207,9 +207,9 @@ type ( Inner1Test `json:"inner1"` Inner2Test `json:"-"` Inner3Test `json:"-,"` - Inner4Test `json:"inner4,omitempty"` + Inner4Test `json:"inner4"` Inner5Test `json:","` - Inner6Test `json:"inner6,omitempty"` + Inner6Test `json:"inner6"` } ) @@ -218,7 +218,7 @@ func TestPretty(t *testing.T) { strPtr := func(s string) *string { return &s } testCases := []struct { - val interface{} + val any exp string // used in testCases where JSON can't handle it }{{ val: "strval", @@ -369,11 +369,11 @@ func TestPretty(t *testing.T) { val: struct { A *int B *int - C interface{} - D interface{} + C any + D any }{ B: intPtr(1), - D: interface{}(2), + D: any(2), }, }, { val: marshalerTest{"foobar"}, @@ -650,13 +650,13 @@ func TestPretty(t *testing.T) { } } -func makeKV(args ...interface{}) []interface{} { return args } +func makeKV(args ...any) []any { return args } func TestRender(t *testing.T) { testCases := []struct { name string - builtins []interface{} - args []interface{} + builtins []any + args []any wantKV string wantJSON string }{{ @@ -665,8 +665,8 @@ func TestRender(t *testing.T) { wantJSON: "{}", }, { name: "empty", - builtins: []interface{}{}, - args: []interface{}{}, + builtins: []any{}, + args: []any{}, wantKV: "", wantJSON: "{}", }, { @@ -739,12 +739,12 @@ func TestRender(t *testing.T) { func TestSanitize(t *testing.T) { testCases := []struct { name string - kv []interface{} - want []interface{} + kv []any + want []any }{{ name: "empty", - kv: []interface{}{}, - want: []interface{}{}, + kv: []any{}, + want: []any{}, }, { name: "already sane", kv: makeKV("int", 1, "str", "ABC", "bool", true), diff --git a/pkg/log/httpaccess/http_access.go b/pkg/log/httpaccess/http_access.go index 668b449813a..fbc350dba60 100644 --- a/pkg/log/httpaccess/http_access.go +++ b/pkg/log/httpaccess/http_access.go @@ -58,7 +58,7 @@ func NewHTTPAccessLogHandler(logger log.Logger, tracer *tracing.Tracer, message ip = r.RemoteAddr } - fields := []interface{}{ + fields := []any{ "ip", ip, "method", r.Method, "host", r.Host, diff --git a/pkg/log/log.go b/pkg/log/log.go index 5a517d02a33..0efa22c6127 100644 --- a/pkg/log/log.go +++ b/pkg/log/log.go @@ -116,7 +116,7 @@ type Builder interface { // WithValues specifies additional key/value pairs // to be logged with each log line. - WithValues(keysAndValues ...interface{}) Builder + WithValues(keysAndValues ...any) Builder // Build returns a new or existing Logger // instance, if such instance already exists. @@ -143,28 +143,28 @@ type Logger interface { // the log line. The key/value pairs can then be used to add additional // variable information. The key/value pairs must alternate string keys // and arbitrary values. - Debug(msg string, keysAndValues ...interface{}) + Debug(msg string, keysAndValues ...any) // Info logs an info message with the given key/value pairs as context. // The msg argument should be used to add some constant description to // the log line. The key/value pairs can then be used to add additional // variable information. The key/value pairs must alternate string keys // and arbitrary values. - Info(msg string, keysAndValues ...interface{}) + Info(msg string, keysAndValues ...any) // Warning logs a warning message with the given key/value pairs as context. // The msg argument should be used to add some constant description to // the log line. The key/value pairs can then be used to add additional // variable information. The key/value pairs must alternate string keys // and arbitrary values. - Warning(msg string, keysAndValues ...interface{}) + Warning(msg string, keysAndValues ...any) // Error logs an error, with the given message and key/value pairs as context. // The msg argument should be used to add context to any underlying error, // while the err argument should be used to attach the actual error that // triggered this log line, if present. The err parameter is optional // and nil may be passed instead of an error instance. - Error(err error, msg string, keysAndValues ...interface{}) + Error(err error, msg string, keysAndValues ...any) } // Lock wraps io.Writer in a mutex to make it safe for concurrent use. diff --git a/pkg/log/logger.go b/pkg/log/logger.go index f40ffd59916..bb81d327172 100644 --- a/pkg/log/logger.go +++ b/pkg/log/logger.go @@ -58,7 +58,7 @@ type builder struct { // values holds additional key/value pairs // that are included on every log call. - values []interface{} + values []any // valuesStr is a cache of render values slice, so // we don't have to render them on each Build call. @@ -83,7 +83,7 @@ func (b *builder) WithName(name string) Builder { } // WithValues implements the Builder interface WithValues method. -func (b *builder) WithValues(keysAndValues ...interface{}) Builder { +func (b *builder) WithValues(keysAndValues ...any) Builder { c := b.clone() c.values = append(c.values, keysAndValues...) return c @@ -133,7 +133,7 @@ func (b *builder) clone() *builder { c := *b c.cloned = true c.names = append(make([]string, 0, len(c.names)), c.names...) - c.values = append(make([]interface{}, 0, len(c.values)), c.values...) + c.values = append(make([]any, 0, len(c.values)), c.values...) return &c } @@ -176,7 +176,7 @@ func (l *logger) Verbosity() Level { } // Debug implements the Logger interface Debug method. -func (l *logger) Debug(msg string, keysAndValues ...interface{}) { +func (l *logger) Debug(msg string, keysAndValues ...any) { if int(l.verbosity.get()) >= int(l.v) { if err := l.log(VerbosityDebug, CategoryDebug, nil, msg, keysAndValues...); err != nil { fmt.Fprintln(os.Stderr, err) @@ -185,7 +185,7 @@ func (l *logger) Debug(msg string, keysAndValues ...interface{}) { } // Info implements the Logger interface Info method. -func (l *logger) Info(msg string, keysAndValues ...interface{}) { +func (l *logger) Info(msg string, keysAndValues ...any) { if l.verbosity.get() >= VerbosityInfo { if err := l.log(VerbosityInfo, CategoryInfo, nil, msg, keysAndValues...); err != nil { fmt.Fprintln(os.Stderr, err) @@ -194,7 +194,7 @@ func (l *logger) Info(msg string, keysAndValues ...interface{}) { } // Warning implements the Logger interface Warning method. -func (l *logger) Warning(msg string, keysAndValues ...interface{}) { +func (l *logger) Warning(msg string, keysAndValues ...any) { if l.verbosity.get() >= VerbosityWarning { if err := l.log(VerbosityWarning, CategoryWarning, nil, msg, keysAndValues...); err != nil { fmt.Fprintln(os.Stderr, err) @@ -203,7 +203,7 @@ func (l *logger) Warning(msg string, keysAndValues ...interface{}) { } // Error implements the Logger interface Error method. -func (l *logger) Error(err error, msg string, keysAndValues ...interface{}) { +func (l *logger) Error(err error, msg string, keysAndValues ...any) { if l.verbosity.get() >= VerbosityError { if err := l.log(VerbosityError, CategoryError, err, msg, keysAndValues...); err != nil { fmt.Fprintln(os.Stderr, err) @@ -218,8 +218,8 @@ func (l *logger) setVerbosity(v Level) { // log logs the given msg and key-value pairs with the given level // and the given message category caller (if enabled) to the sink. -func (l *logger) log(vl Level, mc MessageCategory, err error, msg string, keysAndValues ...interface{}) error { - base := make([]interface{}, 0, 14+len(keysAndValues)) +func (l *logger) log(vl Level, mc MessageCategory, err error, msg string, keysAndValues ...any) error { + base := make([]any, 0, 14+len(keysAndValues)) if l.formatter.opts.logTimestamp { base = append(base, "time", time.Now().Format(l.formatter.opts.timestampLayout)) } diff --git a/pkg/log/logger_test.go b/pkg/log/logger_test.go index f7c23308808..e3921b60a84 100644 --- a/pkg/log/logger_test.go +++ b/pkg/log/logger_test.go @@ -30,8 +30,8 @@ func (h *hook) Fire(Level) error { } // applyError is a higher order function that returns the given fn with an applied err. -func applyError(fn func(error, string, ...interface{}), err error) func(string, ...interface{}) { - return func(msg string, kvs ...interface{}) { +func applyError(fn func(error, string, ...any), err error) func(string, ...any) { + return func(msg string, kvs ...any) { fn(err, msg, kvs...) } } @@ -73,7 +73,7 @@ func TestLoggerOptionsLevelHooks(t *testing.T) { logger, _ := newLogger(WithLevelHooks(VerbosityNone, &have)) tests := []struct { - fn func(string, ...interface{}) + fn func(string, ...any) want bool }{{ fn: logger.Build().Debug, @@ -102,7 +102,7 @@ func TestLoggerOptionsLevelHooks(t *testing.T) { logger, _ := newLogger(WithLevelHooks(VerbosityDebug, &have)) tests := []struct { - fn func(string, ...interface{}) + fn func(string, ...any) want bool }{{ fn: logger.Build().Debug, @@ -131,7 +131,7 @@ func TestLoggerOptionsLevelHooks(t *testing.T) { logger, _ := newLogger(WithLevelHooks(VerbosityInfo, &have)) tests := []struct { - fn func(string, ...interface{}) + fn func(string, ...any) want bool }{{ fn: logger.Build().Debug, @@ -160,7 +160,7 @@ func TestLoggerOptionsLevelHooks(t *testing.T) { logger, _ := newLogger(WithLevelHooks(VerbosityWarning, &have)) tests := []struct { - fn func(string, ...interface{}) + fn func(string, ...any) want bool }{{ fn: logger.Build().Debug, @@ -189,7 +189,7 @@ func TestLoggerOptionsLevelHooks(t *testing.T) { logger, _ := newLogger(WithLevelHooks(VerbosityError, &have)) tests := []struct { - fn func(string, ...interface{}) + fn func(string, ...any) want bool }{{ fn: logger.Build().Debug, @@ -218,7 +218,7 @@ func TestLoggerOptionsLevelHooks(t *testing.T) { logger, _ := newLogger(WithLevelHooks(VerbosityAll, &have)) tests := []struct { - fn func(string, ...interface{}) + fn func(string, ...any) want bool }{{ fn: logger.Build().Debug, @@ -275,8 +275,8 @@ func TestLogger(t *testing.T) { testCases := []struct { name string - logFn func(string, ...interface{}) - args []interface{} + logFn func(string, ...any) + args []any want string }{{ name: "just msg", @@ -626,8 +626,8 @@ func TestLoggerWithName(t *testing.T) { testCases := []struct { name string - logFn func(string, ...interface{}) - args []interface{} + logFn func(string, ...any) + args []any want string }{{ name: "one", @@ -694,8 +694,8 @@ func TestLoggerWithValues(t *testing.T) { testCases := []struct { name string - logFn func(string, ...interface{}) - args []interface{} + logFn func(string, ...any) + args []any want string }{{ name: "zero", diff --git a/pkg/log/noop_logger.go b/pkg/log/noop_logger.go index 590a717d722..96fbddefd34 100644 --- a/pkg/log/noop_logger.go +++ b/pkg/log/noop_logger.go @@ -9,14 +9,14 @@ var Noop Logger = new(noopLogger) type noopLogger struct{} -func (nl *noopLogger) V(_ uint) Builder { return nl } -func (nl *noopLogger) WithName(_ string) Builder { return nl } -func (nl *noopLogger) WithValues(_ ...interface{}) Builder { return nl } -func (nl *noopLogger) Build() Logger { return nl } -func (nl *noopLogger) Register() Logger { return nl } +func (nl *noopLogger) V(_ uint) Builder { return nl } +func (nl *noopLogger) WithName(_ string) Builder { return nl } +func (nl *noopLogger) WithValues(_ ...any) Builder { return nl } +func (nl *noopLogger) Build() Logger { return nl } +func (nl *noopLogger) Register() Logger { return nl } -func (nl *noopLogger) Verbosity() Level { return VerbosityNone } -func (nl *noopLogger) Debug(_ string, _ ...interface{}) {} -func (nl *noopLogger) Info(_ string, _ ...interface{}) {} -func (nl *noopLogger) Warning(_ string, _ ...interface{}) {} -func (nl *noopLogger) Error(_ error, _ string, _ ...interface{}) {} +func (nl *noopLogger) Verbosity() Level { return VerbosityNone } +func (nl *noopLogger) Debug(_ string, _ ...any) {} +func (nl *noopLogger) Info(_ string, _ ...any) {} +func (nl *noopLogger) Warning(_ string, _ ...any) {} +func (nl *noopLogger) Error(_ error, _ string, _ ...any) {} diff --git a/pkg/log/registry.go b/pkg/log/registry.go index a467cfc39cf..bd7fa3587c6 100644 --- a/pkg/log/registry.go +++ b/pkg/log/registry.go @@ -124,7 +124,7 @@ func SetVerbosityByExp(e string, v Level) error { } var merr *multierror.Error - loggers.Range(func(key, val interface{}) bool { + loggers.Range(func(key, val any) bool { if rex.MatchString(key.(string)) { merr = multierror.Append(merr, SetVerbosity(val.(*logger), v)) } @@ -135,7 +135,7 @@ func SetVerbosityByExp(e string, v Level) error { // RegistryIterate iterates through all registered loggers. func RegistryIterate(fn func(id, path string, verbosity Level, v uint) (next bool)) { - loggers.Range(func(_, val interface{}) bool { + loggers.Range(func(_, val any) bool { l := val.(*logger) return fn(l.id, l.namesStr, l.verbosity.get(), l.v) }) diff --git a/pkg/log/registry_test.go b/pkg/log/registry_test.go index dbfb1410d84..e83ac0bc588 100644 --- a/pkg/log/registry_test.go +++ b/pkg/log/registry_test.go @@ -75,10 +75,10 @@ func TestNewLogger(t *testing.T) { var ( cnt int - val interface{} + val any ) NewLogger("root").Register() - loggers.Range(func(k, v interface{}) bool { + loggers.Range(func(k, v any) bool { cnt++ val = v return true @@ -119,7 +119,7 @@ func TestSetVerbosity(t *testing.T) { NewLogger("root").WithName("child1").WithValues("abc", 123).Register() registered := make(map[string]*logger) - loggers.Range(func(k, v interface{}) bool { + loggers.Range(func(k, v any) bool { registered[k.(string)] = v.(*logger) return true }) @@ -185,7 +185,7 @@ func TestRegistryRange(t *testing.T) { NewLogger("root").WithName("child1").WithValues("abc", 123).Register() registered := make(map[string]*logger) - loggers.Range(func(k, v interface{}) bool { + loggers.Range(func(k, v any) bool { registered[k.(string)] = v.(*logger) return true }) diff --git a/pkg/manifest/mantaray/marshal.go b/pkg/manifest/mantaray/marshal.go index f5f7edabc8f..6d355721c1c 100644 --- a/pkg/manifest/mantaray/marshal.go +++ b/pkg/manifest/mantaray/marshal.go @@ -447,7 +447,7 @@ func nodeRefBytes(f *fork) []byte { func encryptDecrypt(input, key []byte) []byte { output := make([]byte, len(input)) - for i := 0; i < len(input); i++ { + for i := range input { output[i] = input[i] ^ key[i%len(key)] } diff --git a/pkg/manifest/mantaray/marshal_test.go b/pkg/manifest/mantaray/marshal_test.go index e0d728602fa..8d0cb827b84 100644 --- a/pkg/manifest/mantaray/marshal_test.go +++ b/pkg/manifest/mantaray/marshal_test.go @@ -177,7 +177,7 @@ func TestMarshal(t *testing.T) { i++ return b } - for i := 0; i < len(testEntries); i++ { + for i := range testEntries { c := testEntries[i].Path e := testEntries[i].Entry if len(e) == 0 { diff --git a/pkg/manifest/mantaray/node_test.go b/pkg/manifest/mantaray/node_test.go index b9a8da787d7..80b0ed4963b 100644 --- a/pkg/manifest/mantaray/node_test.go +++ b/pkg/manifest/mantaray/node_test.go @@ -41,14 +41,14 @@ func TestAddAndLookup(t *testing.T) { []byte("aa"), []byte("b"), } - for i := 0; i < len(testCases); i++ { + for i := range testCases { c := testCases[i] e := append(make([]byte, 32-len(c)), c...) err := n.Add(ctx, c, e, nil, nil) if err != nil { t.Fatalf("expected no error, got %v", err) } - for j := 0; j < i; j++ { + for j := range i { d := testCases[j] m, err := n.Lookup(ctx, d, nil) if err != nil { diff --git a/pkg/manifest/mantaray/persist_test.go b/pkg/manifest/mantaray/persist_test.go index e0eb86e6b09..619989d3803 100644 --- a/pkg/manifest/mantaray/persist_test.go +++ b/pkg/manifest/mantaray/persist_test.go @@ -32,7 +32,7 @@ func TestPersistIdempotence(t *testing.T) { } ctx := context.Background() var ls mantaray.LoadSaver = newMockLoadSaver() - for i := 0; i < len(paths); i++ { + for i := range paths { c := paths[i] err := n.Save(ctx, ls) if err != nil { @@ -49,7 +49,7 @@ func TestPersistIdempotence(t *testing.T) { if err != nil { t.Fatalf("expected no error, got %v", err) } - for i := 0; i < len(paths); i++ { + for i := range paths { c := paths[i] m, err := n.Lookup(ctx, c, ls) if err != nil { diff --git a/pkg/manifest/mantaray/walker_test.go b/pkg/manifest/mantaray/walker_test.go index f6d72687a39..76c23f96e69 100644 --- a/pkg/manifest/mantaray/walker_test.go +++ b/pkg/manifest/mantaray/walker_test.go @@ -55,7 +55,7 @@ func TestWalkNode(t *testing.T) { n := mantaray.New() - for i := 0; i < len(toAdd); i++ { + for i := range toAdd { c := toAdd[i] e := append(make([]byte, 32-len(c)), c...) err := n.Add(ctx, c, e, nil, nil) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index cbe2ae20c58..64ceef0c51e 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -18,7 +18,7 @@ type Collector interface { Metrics() []prometheus.Collector } -func PrometheusCollectorsFromFields(i interface{}) (cs []prometheus.Collector) { +func PrometheusCollectorsFromFields(i any) (cs []prometheus.Collector) { v := reflect.Indirect(reflect.ValueOf(i)) for i := 0; i < v.NumField(); i++ { if !v.Field(i).CanInterface() { diff --git a/pkg/node/bootstrap.go b/pkg/node/bootstrap.go index 4ce709b16d9..047c306aaf2 100644 --- a/pkg/node/bootstrap.go +++ b/pkg/node/bootstrap.go @@ -204,7 +204,7 @@ func bootstrapNode( eventsJSON []byte ) - for i := 0; i < getSnapshotRetries; i++ { + for range getSnapshotRetries { if err != nil { time.Sleep(retryWait) } @@ -223,7 +223,7 @@ func bootstrapNode( return nil, err } - for i := 0; i < getSnapshotRetries; i++ { + for range getSnapshotRetries { if err != nil { time.Sleep(retryWait) } diff --git a/pkg/p2p/libp2p/connections_test.go b/pkg/p2p/libp2p/connections_test.go index 51227ab1555..93d8ab41740 100644 --- a/pkg/p2p/libp2p/connections_test.go +++ b/pkg/p2p/libp2p/connections_test.go @@ -126,7 +126,7 @@ func TestLightPeerLimit(t *testing.T) { addr := serviceUnderlayAddress(t, sf) - for i := 0; i < 5; i++ { + for range 5 { sl, _ := newService(t, 1, libp2pServiceOpts{ notifier: notifier, libp2pOpts: libp2p.Options{ @@ -240,7 +240,7 @@ func TestStreamsMaxIncomingLimit(t *testing.T) { // close random streams to validate new streams creation random := rand.New(rand.NewSource(time.Now().UnixNano())) - for i := 0; i < closeStreamCount; i++ { + for range closeStreamCount { n := random.Intn(len(streams)) if err := streams[n].Reset(); err != nil { t.Error(err) @@ -840,7 +840,7 @@ func TestTopologyAnnounce(t *testing.T) { expectPeersEventually(t, s1, overlay3) called := false - for i := 0; i < 20; i++ { + for range 20 { mtx.Lock() called = announceCalled mtx.Unlock() @@ -852,7 +852,7 @@ func TestTopologyAnnounce(t *testing.T) { if !called { t.Error("expected announce to be called") } - for i := 0; i < 10; i++ { + for range 10 { mtx.Lock() called = announceToCalled mtx.Unlock() @@ -878,7 +878,7 @@ func TestTopologyAnnounce(t *testing.T) { expectPeers(t, s2, overlay1) expectPeersEventually(t, s1, overlay2, overlay3) - for i := 0; i < 20; i++ { + for range 20 { mtx.Lock() called = announceToCalled mtx.Unlock() diff --git a/pkg/p2p/libp2p/libp2p.go b/pkg/p2p/libp2p/libp2p.go index 8020b381708..e1091e0b637 100644 --- a/pkg/p2p/libp2p/libp2p.go +++ b/pkg/p2p/libp2p/libp2p.go @@ -367,7 +367,7 @@ func New(ctx context.Context, signer beecrypto.Signer, networkID uint64, overlay } func (s *Service) reachabilityWorker() error { - sub, err := s.host.EventBus().Subscribe([]interface{}{new(event.EvtLocalReachabilityChanged)}) + sub, err := s.host.EventBus().Subscribe([]any{new(event.EvtLocalReachabilityChanged)}) if err != nil { return fmt.Errorf("failed subscribing to reachability event %w", err) } @@ -1060,7 +1060,7 @@ func (s *Service) peerUserAgent(ctx context.Context, peerID libp2ppeer.ID) strin ctx, cancel := context.WithTimeout(ctx, peerUserAgentTimeout) defer cancel() var ( - v interface{} + v any err error ) // Peerstore may not contain all keys and values right after the connections is created. diff --git a/pkg/p2p/protobuf/protobuf_test.go b/pkg/p2p/protobuf/protobuf_test.go index 3a1edf30bf5..f70e3754161 100644 --- a/pkg/p2p/protobuf/protobuf_test.go +++ b/pkg/p2p/protobuf/protobuf_test.go @@ -49,7 +49,7 @@ func TestReader_ReadMsg(t *testing.T) { r := tc.readerFunc() var msg pb.Message - for i := 0; i < len(messages); i++ { + for i := range messages { err := r.ReadMsg(&msg) if i == len(messages) { if !errors.Is(err, io.EOF) { @@ -105,7 +105,7 @@ func TestReader_timeout(t *testing.T) { r := tc.readerFunc() var msg pb.Message - for i := 0; i < len(messages); i++ { + for i := range messages { var timeout time.Duration if i == 0 { timeout = 1000 * time.Millisecond diff --git a/pkg/p2p/streamtest/streamtest.go b/pkg/p2p/streamtest/streamtest.go index ae312624149..92e538d83bd 100644 --- a/pkg/p2p/streamtest/streamtest.go +++ b/pkg/p2p/streamtest/streamtest.go @@ -348,10 +348,7 @@ func (r *record) Read(p []byte) (n int, err error) { r.lock.Lock() defer r.lock.Unlock() - end := r.c + len(p) - if end > len(r.b) { - end = len(r.b) - } + end := min(r.c+len(p), len(r.b)) n = copy(p, r.b[r.c:end]) r.c += n diff --git a/pkg/p2p/streamtest/streamtest_test.go b/pkg/p2p/streamtest/streamtest_test.go index 0b26d2da7ce..7dd1ae02091 100644 --- a/pkg/p2p/streamtest/streamtest_test.go +++ b/pkg/p2p/streamtest/streamtest_test.go @@ -813,7 +813,7 @@ func testRecords(t *testing.T, records []*streamtest.Record, want [][2]string, w t.Fatalf("got %v records, want %v", lr, lw) } - for i := 0; i < lr; i++ { + for i := range lr { record := records[i] if err := record.Err(); !errors.Is(err, wantErr) { diff --git a/pkg/postage/batchstore/mock/store_test.go b/pkg/postage/batchstore/mock/store_test.go index d376c359819..8100da0dac7 100644 --- a/pkg/postage/batchstore/mock/store_test.go +++ b/pkg/postage/batchstore/mock/store_test.go @@ -32,7 +32,7 @@ func TestBatchStore(t *testing.T) { } // Update should return error after a number of tries: - for i := 0; i < testCnt; i++ { + for range testCnt { if err := batchStore.Update(testBatch, big.NewInt(0), 0); err != nil { t.Fatal(err) } @@ -45,7 +45,7 @@ func TestBatchStore(t *testing.T) { if _, err := batchStore.Get(postagetesting.MustNewID()); err == nil { t.Fatal("expected error") } - for i := 0; i < testCnt-1; i++ { + for range testCnt - 1 { if _, err := batchStore.Get(testBatch.ID); err != nil { t.Fatal(err) } @@ -67,7 +67,7 @@ func TestBatchStorePutChainState(t *testing.T) { ) // PutChainState should return an error after a number of tries: - for i := 0; i < testCnt; i++ { + for range testCnt { if err := batchStore.PutChainState(testChainState); err != nil { t.Fatal(err) } diff --git a/pkg/postage/batchstore/store_test.go b/pkg/postage/batchstore/store_test.go index 603dd24f398..975c93f3e18 100644 --- a/pkg/postage/batchstore/store_test.go +++ b/pkg/postage/batchstore/store_test.go @@ -602,7 +602,7 @@ func addBatch(t *testing.T, s postage.Storer, depth uint8, value int) *postage.B return batch } -func stateStoreGet(t *testing.T, st storage.StateStorer, k string, v interface{}) { +func stateStoreGet(t *testing.T, st storage.StateStorer, k string, v any) { t.Helper() if err := st.Get(k, v); err != nil { @@ -610,7 +610,7 @@ func stateStoreGet(t *testing.T, st storage.StateStorer, k string, v interface{} } } -func stateStorePut(t *testing.T, st storage.StateStorer, k string, v interface{}) { +func stateStorePut(t *testing.T, st storage.StateStorer, k string, v any) { t.Helper() if err := st.Put(k, v); err != nil { diff --git a/pkg/postage/listener/listener_test.go b/pkg/postage/listener/listener_test.go index e3ee957a3e8..bd15c2659ab 100644 --- a/pkg/postage/listener/listener_test.go +++ b/pkg/postage/listener/listener_test.go @@ -536,19 +536,19 @@ func TestListenerBatchState(t *testing.T) { func newEventUpdaterMock() *updater { return &updater{ - eventC: make(chan interface{}, 1), + eventC: make(chan any, 1), } } func newEventUpdaterMockWithBlockNumberUpdateError(err error) *updater { return &updater{ - eventC: make(chan interface{}, 1), + eventC: make(chan any, 1), blockNumberUpdateError: err, } } type updater struct { - eventC chan interface{} + eventC chan any blockNumberUpdateError error } diff --git a/pkg/postage/service_test.go b/pkg/postage/service_test.go index 1ae9dfd1b46..1237fecc066 100644 --- a/pkg/postage/service_test.go +++ b/pkg/postage/service_test.go @@ -36,7 +36,7 @@ func TestSaveLoad(t *testing.T) { if err != nil { t.Fatal(err) } - for i := 0; i < 16; i++ { + for range 16 { err := ps.Add(newTestStampIssuer(t, 1000)) if err != nil { t.Fatal(err) diff --git a/pkg/postage/stamper_test.go b/pkg/postage/stamper_test.go index 383c534a0c4..f362ee8e23c 100644 --- a/pkg/postage/stamper_test.go +++ b/pkg/postage/stamper_test.go @@ -72,7 +72,7 @@ func TestStamperStamping(t *testing.T) { chunkAddr, _ := createStamp(t, stamper) // issue another 15 // collision depth is 8, committed batch depth is 12, bucket volume 2^4 - for i := 0; i < 14; i++ { + for i := range 14 { randAddr := swarm.RandAddressAt(t, chunkAddr, 8) _, err = stamper.Stamp(randAddr, randAddr) if err != nil { @@ -98,7 +98,7 @@ func TestStamperStamping(t *testing.T) { chunkAddr, _ := createStamp(t, stamper) // issue another 15 // collision depth is 8, committed batch depth is 12, bucket volume 2^4 - for i := 0; i < 15; i++ { + for i := range 15 { randAddr := swarm.RandAddressAt(t, chunkAddr, 8) _, err = stamper.Stamp(randAddr, randAddr) if err != nil { diff --git a/pkg/postage/stampissuer_test.go b/pkg/postage/stampissuer_test.go index 5e7bfa71cbc..05c177360a7 100644 --- a/pkg/postage/stampissuer_test.go +++ b/pkg/postage/stampissuer_test.go @@ -176,7 +176,7 @@ func Test_StampIssuer_inc(t *testing.T) { count := sti.BucketUpperBound() // Increment to upper bound (fill bucket to max cap) - for i := uint32(0); i < count; i++ { + for range count { _, _, err := sti.Increment(addr) if err != nil { t.Fatal(err) @@ -184,7 +184,7 @@ func Test_StampIssuer_inc(t *testing.T) { } // Incrementing stamp issuer above upper bound should return index starting from 0 - for i := uint32(0); i < count; i++ { + for i := range count { idxb, _, err := sti.Increment(addr) if err != nil { t.Fatal(err) @@ -203,7 +203,7 @@ func Test_StampIssuer_inc(t *testing.T) { count := sti.BucketUpperBound() // Increment to upper bound (fill bucket to max cap) - for i := uint32(0); i < count; i++ { + for range count { _, _, err := sti.Increment(addr) if err != nil { t.Fatal(err) @@ -211,7 +211,7 @@ func Test_StampIssuer_inc(t *testing.T) { } // Incrementing stamp issuer above upper bound should return error - for i := uint32(0); i < count; i++ { + for range count { _, _, err := sti.Increment(addr) if !errors.Is(err, postage.ErrBucketFull) { t.Fatal("bucket should be full") @@ -230,7 +230,7 @@ func TestUtilization(t *testing.T) { var eg errgroup.Group - for i := 0; i < 8; i++ { + for range 8 { eg.Go(func() error { for { _, _, err := sti.Increment(swarm.RandAddress(t)) diff --git a/pkg/pss/mining_test.go b/pkg/pss/mining_test.go index 1cb4cc03282..94e469bf6ca 100644 --- a/pkg/pss/mining_test.go +++ b/pkg/pss/mining_test.go @@ -16,7 +16,7 @@ import ( func newTargets(length, depth int) pss.Targets { targets := make([]pss.Target, length) - for i := 0; i < length; i++ { + for i := range length { buf := make([]byte, 8) binary.LittleEndian.PutUint64(buf, uint64(i)) targets[i] = pss.Target(buf[:depth]) diff --git a/pkg/pss/pss.go b/pkg/pss/pss.go index 454f5960129..28319e9a615 100644 --- a/pkg/pss/pss.go +++ b/pkg/pss/pss.go @@ -130,7 +130,7 @@ func (p *pss) Register(topic Topic, handler Handler) (cleanup func()) { defer p.handlersMu.Unlock() h := p.handlers[topic] - for i := 0; i < len(h); i++ { + for i := range h { if h[i] == &handler { p.handlers[topic] = append(h[:i], h[i+1:]...) return diff --git a/pkg/pss/pss_test.go b/pkg/pss/pss_test.go index 685adb59fb9..2482f9b2c7b 100644 --- a/pkg/pss/pss_test.go +++ b/pkg/pss/pss_test.go @@ -219,7 +219,7 @@ func TestRegister(t *testing.T) { func waitHandlerCallback(t *testing.T, msgChan *chan struct{}, count int) { t.Helper() - for received := 0; received < count; received++ { + for range count { select { case <-*msgChan: case <-time.After(1 * time.Second): diff --git a/pkg/pss/trojan.go b/pkg/pss/trojan.go index b6b404f2389..8316ae6932e 100644 --- a/pkg/pss/trojan.go +++ b/pkg/pss/trojan.go @@ -215,7 +215,7 @@ func mine(ctx context.Context, odd bool, f func(nonce []byte) (swarm.Chunk, erro defer cancel() eg, ctx := errgroup.WithContext(ctx) result := make(chan swarm.Chunk, 8) - for i := 0; i < 8; i++ { + for range 8 { eg.Go(func() error { nonce := make([]byte, 32) copy(nonce, initnonce) diff --git a/pkg/puller/puller.go b/pkg/puller/puller.go index 2fdbc24e9cf..d53a45348e4 100644 --- a/pkg/puller/puller.go +++ b/pkg/puller/puller.go @@ -268,7 +268,7 @@ func (p *Puller) syncPeer(ctx context.Context, peer *syncPeer, storageRadius uin if peer.po >= storageRadius { // cancel all bins lower than the storage radius - for bin := uint8(0); bin < storageRadius; bin++ { + for bin := range storageRadius { peer.cancelBin(bin) } diff --git a/pkg/pullsync/pullsync_test.go b/pkg/pullsync/pullsync_test.go index fc80bae137f..d77b130936e 100644 --- a/pkg/pullsync/pullsync_test.go +++ b/pkg/pullsync/pullsync_test.go @@ -44,7 +44,7 @@ func init() { chunks = make([]swarm.Chunk, n) addrs = make([]swarm.Address, n) results = make([]*storer.BinC, n) - for i := 0; i < n; i++ { + for i := range n { chunks[i] = testingc.GenerateTestRandomChunk() addrs[i] = chunks[i].Address() stampHash, _ := chunks[i].Stamp().Hash() diff --git a/pkg/replicas/getter.go b/pkg/replicas/getter.go index 5dc42cb7c82..b08b5c780e8 100644 --- a/pkg/replicas/getter.go +++ b/pkg/replicas/getter.go @@ -60,9 +60,7 @@ func (g *getter) Get(ctx context.Context, addr swarm.Address) (ch swarm.Chunk, e errcnt := 0 // concurrently call to retrieve chunk using original CAC address - g.wg.Add(1) - go func() { - defer g.wg.Done() + g.wg.Go(func() { ch, err := g.Getter.Get(ctx, addr) if err != nil { errc <- err @@ -73,7 +71,7 @@ func (g *getter) Get(ctx context.Context, addr swarm.Address) (ch swarm.Chunk, e case resultC <- ch: case <-ctx.Done(): } - }() + }) // counters n := 0 // counts the replica addresses tried target := 2 // the number of replicas attempted to download in this batch @@ -117,9 +115,7 @@ func (g *getter) Get(ctx context.Context, addr swarm.Address) (ch swarm.Chunk, e continue } - g.wg.Add(1) - go func() { - defer g.wg.Done() + g.wg.Go(func() { ch, err := g.Getter.Get(ctx, swarm.NewAddress(so.addr)) if err != nil { errc <- err @@ -136,7 +132,7 @@ func (g *getter) Get(ctx context.Context, addr swarm.Address) (ch swarm.Chunk, e case resultC <- soc.WrappedChunk(): case <-ctx.Done(): } - }() + }) n++ if n < target { continue diff --git a/pkg/replicas/putter.go b/pkg/replicas/putter.go index 8e23059e1cf..7614dee56d0 100644 --- a/pkg/replicas/putter.go +++ b/pkg/replicas/putter.go @@ -43,15 +43,13 @@ func (p *putter) Put(ctx context.Context, ch swarm.Chunk) (err error) { errc := make(chan error, p.rLevel.GetReplicaCount()) wg := sync.WaitGroup{} for r := range rr.c { - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { sch, err := soc.New(r.id, ch).Sign(signer) if err == nil { err = p.putter.Put(ctx, sch) } errc <- err - }() + }) } wg.Wait() diff --git a/pkg/salud/salud.go b/pkg/salud/salud.go index 28fe648d030..1cf4ffa8373 100644 --- a/pkg/salud/salud.go +++ b/pkg/salud/salud.go @@ -143,9 +143,7 @@ func (s *service) salud(mode string, durPercentile float64, connsPercentile floa ) err := s.topology.EachConnectedPeer(func(addr swarm.Address, bin uint8) (stop bool, jumpToNext bool, err error) { - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { ctx, cancel := context.WithTimeout(context.Background(), requestTimeout) defer cancel() @@ -172,7 +170,7 @@ func (s *service) salud(mode string, durPercentile float64, connsPercentile floa neighborhoodTotalDur += dur.Seconds() } mtx.Unlock() - }() + }) return false, false, nil }, topology.Select{}) if err != nil { diff --git a/pkg/sharky/recovery.go b/pkg/sharky/recovery.go index 2b19ee93935..af064fda60a 100644 --- a/pkg/sharky/recovery.go +++ b/pkg/sharky/recovery.go @@ -30,7 +30,7 @@ func NewRecovery(dir string, shardCnt int, datasize int) (*Recovery, error) { shards := make([]*slots, shardCnt) shardFiles := make([]*os.File, shardCnt) - for i := 0; i < shardCnt; i++ { + for i := range shardCnt { file, err := os.OpenFile(path.Join(dir, fmt.Sprintf("shard_%03d", i)), os.O_RDWR, 0666) if errors.Is(err, fs.ErrNotExist) { return nil, fmt.Errorf("index %d: %w", i, ErrShardNotFound) diff --git a/pkg/sharky/recovery_test.go b/pkg/sharky/recovery_test.go index 1cc7ae3d55e..73571c5811e 100644 --- a/pkg/sharky/recovery_test.go +++ b/pkg/sharky/recovery_test.go @@ -114,7 +114,7 @@ func TestRecovery(t *testing.T) { defer cancel() runs := 96 - for i := 0; i < runs; i++ { + for range runs { loc, err := s.Write(cctx, payload) if err != nil { if errors.Is(err, context.DeadlineExceeded) { diff --git a/pkg/sharky/shard.go b/pkg/sharky/shard.go index 5073d63bd87..c9047c979a2 100644 --- a/pkg/sharky/shard.go +++ b/pkg/sharky/shard.go @@ -102,11 +102,9 @@ func (sh *shard) process() { defer func() { // this condition checks if an slot is in limbo (popped but not used for write op) if writes != nil { - sh.slots.limboWG.Add(1) - go func() { - defer sh.slots.limboWG.Done() + sh.slots.limboWG.Go(func() { sh.slots.in <- slot - }() + }) } }() free := sh.slots.out diff --git a/pkg/sharky/shard_slots_test.go b/pkg/sharky/shard_slots_test.go index 0a1ab02cf7e..7738a1595aa 100644 --- a/pkg/sharky/shard_slots_test.go +++ b/pkg/sharky/shard_slots_test.go @@ -156,18 +156,14 @@ func newShard(t *testing.T) *shard { terminated := make(chan struct{}) - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { shard.process() close(terminated) - }() + }) - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { slots.process(terminated) - }() + }) return shard } diff --git a/pkg/sharky/sharky_test.go b/pkg/sharky/sharky_test.go index 268a996e7c0..1d4b78d51c4 100644 --- a/pkg/sharky/sharky_test.go +++ b/pkg/sharky/sharky_test.go @@ -185,11 +185,11 @@ func TestConcurrency(t *testing.T) { ctx := context.Background() eg, ectx := errgroup.WithContext(ctx) // a number of workers write sequential numbers to sharky - for k := 0; k < workers; k++ { + for k := range workers { eg.Go(func() error { <-start buf := make([]byte, 4) - for i := 0; i < limit; i++ { + for i := range limit { j := i*workers + k binary.BigEndian.PutUint32(buf, uint32(j)) loc, err := s.Write(ctx, buf) @@ -212,7 +212,7 @@ func TestConcurrency(t *testing.T) { eg.Go(func() error { <-start buf := make([]byte, datasize) - for i := 0; i < limit; i++ { + for range limit { select { case <-ectx.Done(): return ectx.Err() diff --git a/pkg/sharky/slots.go b/pkg/sharky/slots.go index 1f5294f79b6..71d058f2b88 100644 --- a/pkg/sharky/slots.go +++ b/pkg/sharky/slots.go @@ -58,7 +58,7 @@ func (sl *slots) save() error { // extensions are bytewise: can only be multiples of 8 bits func (sl *slots) extend(n int) { sl.size += uint32(n) * 8 - for i := 0; i < n; i++ { + for range n { sl.data = append(sl.data, 0xff) } } @@ -123,12 +123,10 @@ func (sl *slots) process(quit chan struct{}) { out = nil } quit = nil - sl.wg.Add(1) - go func() { - defer sl.wg.Done() + sl.wg.Go(func() { sl.limboWG.Wait() close(sl.in) - }() + }) } } } diff --git a/pkg/sharky/store.go b/pkg/sharky/store.go index 59549ee4a4a..d8947f9d042 100644 --- a/pkg/sharky/store.go +++ b/pkg/sharky/store.go @@ -100,17 +100,13 @@ func (s *Store) create(index uint8, maxDataSize int, basedir fs.FS) (*shard, err quit: s.quit, } terminated := make(chan struct{}) - sh.slots.wg.Add(1) - go func() { - defer sh.slots.wg.Done() + s.wg.Go(func() { sh.process() close(terminated) - }() - sh.slots.wg.Add(1) - go func() { - defer sh.slots.wg.Done() + }) + s.wg.Go(func() { sl.process(terminated) - }() + }) return sh, nil } diff --git a/pkg/shed/example_store_test.go b/pkg/shed/example_store_test.go index 6d77ce907a7..a6adbb2c703 100644 --- a/pkg/shed/example_store_test.go +++ b/pkg/shed/example_store_test.go @@ -238,7 +238,7 @@ func (s *Store) CollectGarbage() (err error) { maxRounds := 10 // arbitrary number, needs to be calculated // Run a few gc rounds. - for roundCount := 0; roundCount < maxRounds; roundCount++ { + for range maxRounds { var garbageCount int // New batch for a new cg round. trash := new(leveldb.Batch) diff --git a/pkg/shed/field_struct.go b/pkg/shed/field_struct.go index 4f909c17b21..9e6f7139800 100644 --- a/pkg/shed/field_struct.go +++ b/pkg/shed/field_struct.go @@ -45,7 +45,7 @@ func (db *DB) NewStructField(name string) (f StructField, err error) { // Get unmarshals data from the database to a provided val. // If the data is not found leveldb.ErrNotFound is returned. -func (f StructField) Get(val interface{}) (err error) { +func (f StructField) Get(val any) (err error) { b, err := f.db.Get(f.key) if err != nil { return err @@ -54,7 +54,7 @@ func (f StructField) Get(val interface{}) (err error) { } // Put marshals provided val and saves it to the database. -func (f StructField) Put(val interface{}) (err error) { +func (f StructField) Put(val any) (err error) { b, err := json.Marshal(val) if err != nil { return err @@ -63,7 +63,7 @@ func (f StructField) Put(val interface{}) (err error) { } // PutInBatch marshals provided val and puts it into the batch. -func (f StructField) PutInBatch(batch *leveldb.Batch, val interface{}) (err error) { +func (f StructField) PutInBatch(batch *leveldb.Batch, val any) (err error) { b, err := json.Marshal(val) if err != nil { return err diff --git a/pkg/statestore/leveldb/leveldb.go b/pkg/statestore/leveldb/leveldb.go index 08d757ce288..8ddd71d0215 100644 --- a/pkg/statestore/leveldb/leveldb.go +++ b/pkg/statestore/leveldb/leveldb.go @@ -76,7 +76,7 @@ func NewStateStore(path string, l log.Logger) (*Store, error) { // Get retrieves a value of the requested key. If no results are found, // storage.ErrNotFound will be returned. -func (s *Store) Get(key string, i interface{}) error { +func (s *Store) Get(key string, i any) error { data, err := s.db.Get([]byte(key), nil) if err != nil { if errors.Is(err, leveldb.ErrNotFound) { @@ -95,7 +95,7 @@ func (s *Store) Get(key string, i interface{}) error { // Put stores a value for an arbitrary key. BinaryMarshaler // interface method will be called on the provided value // with fallback to JSON serialization. -func (s *Store) Put(key string, i interface{}) (err error) { +func (s *Store) Put(key string, i any) (err error) { var bytes []byte if marshaler, ok := i.(encoding.BinaryMarshaler); ok { if bytes, err = marshaler.MarshalBinary(); err != nil { diff --git a/pkg/statestore/mock/store.go b/pkg/statestore/mock/store.go index a3d34542f2c..7c30fca12b5 100644 --- a/pkg/statestore/mock/store.go +++ b/pkg/statestore/mock/store.go @@ -28,7 +28,7 @@ func NewStateStore() storage.StateStorer { return s } -func (s *store) Get(key string, i interface{}) (err error) { +func (s *store) Get(key string, i any) (err error) { s.mtx.RLock() defer s.mtx.RUnlock() @@ -44,7 +44,7 @@ func (s *store) Get(key string, i interface{}) (err error) { return json.Unmarshal(data, i) } -func (s *store) Put(key string, i interface{}) (err error) { +func (s *store) Put(key string, i any) (err error) { s.mtx.Lock() defer s.mtx.Unlock() diff --git a/pkg/statestore/storeadapter/storeadapter.go b/pkg/statestore/storeadapter/storeadapter.go index 1deda6c6cd3..95e775bffb2 100644 --- a/pkg/statestore/storeadapter/storeadapter.go +++ b/pkg/statestore/storeadapter/storeadapter.go @@ -25,7 +25,7 @@ var _ storage.Item = (*proxyItem)(nil) type proxyItem struct { ns string key string - obj interface{} + obj any } // ID implements Item interface. @@ -91,7 +91,7 @@ func (pi proxyItem) String() string { } // newProxyItem creates a new proxyItem. -func newProxyItem(key string, obj interface{}) *proxyItem { +func newProxyItem(key string, obj any) *proxyItem { return &proxyItem{ns: stateStoreNamespace, key: key, obj: obj} } @@ -146,12 +146,12 @@ func (s *StateStorerAdapter) Close() error { } // Get implements StateStorer interface. -func (s *StateStorerAdapter) Get(key string, obj interface{}) (err error) { +func (s *StateStorerAdapter) Get(key string, obj any) (err error) { return s.storage.Get(newProxyItem(key, obj)) } // Put implements StateStorer interface. -func (s *StateStorerAdapter) Put(key string, obj interface{}) (err error) { +func (s *StateStorerAdapter) Put(key string, obj any) (err error) { return s.storage.Put(newProxyItem(key, obj)) } diff --git a/pkg/statestore/test/store.go b/pkg/statestore/test/store.go index 10bc9c15c83..48cb487edfe 100644 --- a/pkg/statestore/test/store.go +++ b/pkg/statestore/test/store.go @@ -159,7 +159,7 @@ func insertValues(t *testing.T, store storage.StateStorer, key1, key2 string, va func insert(t *testing.T, store storage.StateStorer, prefix string, count int) { t.Helper() - for i := 0; i < count; i++ { + for i := range count { k := prefix + fmt.Sprint(i) err := store.Put(k, i) diff --git a/pkg/storage/inmemstore/inmemstore.go b/pkg/storage/inmemstore/inmemstore.go index b0052602aa1..88076df2722 100644 --- a/pkg/storage/inmemstore/inmemstore.go +++ b/pkg/storage/inmemstore/inmemstore.go @@ -104,7 +104,7 @@ func (s *Store) Count(k storage.Key) (int, error) { defer s.mu.RUnlock() count := 0 - s.st.WalkPrefix(k.Namespace(), func(_ string, _ interface{}) bool { + s.st.WalkPrefix(k.Namespace(), func(_ string, _ any) bool { count++ return false }) @@ -116,7 +116,7 @@ func (s *Store) Iterate(q storage.Query, fn storage.IterateFn) error { return fmt.Errorf("failed iteration: %w", err) } - getNext := func(k string, v interface{}) (*storage.Result, error) { + getNext := func(k string, v any) (*storage.Result, error) { for _, filter := range q.Filters { if filter(idFromKey(k, q.Factory().Namespace()), v.([]byte)) { return nil, nil @@ -155,7 +155,7 @@ func (s *Store) Iterate(q storage.Query, fn storage.IterateFn) error { switch q.Order { case storage.KeyAscendingOrder: - s.st.WalkPrefix(prefix, func(k string, v interface{}) bool { + s.st.WalkPrefix(prefix, func(k string, v any) bool { if q.PrefixAtStart && !skipUntil { if k >= prefix+separator+q.Prefix { @@ -195,7 +195,7 @@ func (s *Store) Iterate(q storage.Query, fn storage.IterateFn) error { // For now, inmem implementation is not meant to work for large datasets, so first option // is chosen. results := make([]storage.Result, 0) - s.st.WalkPrefix(prefix, func(k string, v interface{}) bool { + s.st.WalkPrefix(prefix, func(k string, v any) bool { res, err := getNext(k, v) if err != nil { retErr = errors.Join(retErr, err) diff --git a/pkg/storage/migration/index_test.go b/pkg/storage/migration/index_test.go index ec35859d01f..2247c36ebef 100644 --- a/pkg/storage/migration/index_test.go +++ b/pkg/storage/migration/index_test.go @@ -326,7 +326,7 @@ func TestOptions(t *testing.T) { func populateStore(t *testing.T, s storage.Store, count int) { t.Helper() - for i := 0; i < count; i++ { + for i := range count { item := &obj{id: i, val: i} if err := s.Put(item); err != nil { t.Fatalf("populate store should succeed: %v", err) diff --git a/pkg/storage/migration/steps_chain_test.go b/pkg/storage/migration/steps_chain_test.go index e5a69917f65..327a41331d7 100644 --- a/pkg/storage/migration/steps_chain_test.go +++ b/pkg/storage/migration/steps_chain_test.go @@ -22,7 +22,7 @@ func TestNewStepsChain(t *testing.T) { stepsFn := make([]migration.StepFn, 0) // Create 10 step functions where each would remove single element, having value [0-10) - for i := 0; i < 10; i++ { + for i := range 10 { valForRemoval := i var stepFn migration.StepFn diff --git a/pkg/storage/statestore.go b/pkg/storage/statestore.go index 53520dea1be..b04d36d3551 100644 --- a/pkg/storage/statestore.go +++ b/pkg/storage/statestore.go @@ -16,10 +16,10 @@ type StateStorer interface { io.Closer // Get unmarshalls object with the given key into the given obj. - Get(key string, obj interface{}) error + Get(key string, obj any) error // Put inserts or updates the given obj stored under the given key. - Put(key string, obj interface{}) error + Put(key string, obj any) error // Delete removes object form the store stored under the given key. Delete(key string) error diff --git a/pkg/storage/storagetest/benchmark.go b/pkg/storage/storagetest/benchmark.go index fe2fe3dec12..8d051a06391 100644 --- a/pkg/storage/storagetest/benchmark.go +++ b/pkg/storage/storagetest/benchmark.go @@ -37,7 +37,7 @@ const ( func randomBytes(r *rand.Rand, n int) []byte { b := make([]byte, n) - for i := 0; i < n; i++ { + for i := range n { b[i] = ' ' + byte(r.Intn('~'-' '+1)) } return b @@ -104,7 +104,7 @@ func newStartAtEntryGenerator(start int, g entryGenerator) entryGenerator { func newSequentialKeys(size int, start int, keyFormat string) [][]byte { keys := make([][]byte, size) buffer := make([]byte, size*keyLen) - for i := 0; i < size; i++ { + for i := range size { begin, end := i*keyLen, (i+1)*keyLen key := buffer[begin:begin:end] _, _ = fmt.Fprintf(bytes.NewBuffer(key), keyFormat, start+i) @@ -117,7 +117,7 @@ func newRandomKeys(n int, format string) [][]byte { r := rand.New(rand.NewSource(time.Now().Unix())) keys := make([][]byte, n) buffer := make([]byte, n*keyLen) - for i := 0; i < n; i++ { + for i := range n { begin, end := i*keyLen, (i+1)*keyLen key := buffer[begin:begin:end] _, _ = fmt.Fprintf(bytes.NewBuffer(key), format, r.Intn(n)) @@ -129,7 +129,7 @@ func newRandomKeys(n int, format string) [][]byte { func newFullRandomKeys(size int, start int, format string) [][]byte { keys := newSequentialKeys(size, start, format) r := rand.New(rand.NewSource(time.Now().Unix())) - for i := 0; i < size; i++ { + for i := range size { j := r.Intn(size) keys[i], keys[j] = keys[j], keys[i] } diff --git a/pkg/storage/storagetest/storage.go b/pkg/storage/storagetest/storage.go index e40e6bb9f20..9ee41971b43 100644 --- a/pkg/storage/storagetest/storage.go +++ b/pkg/storage/storagetest/storage.go @@ -984,7 +984,7 @@ func BenchmarkWriteRandom(b *testing.B, db storage.Store) { start, step := 0, (b.N+parallelism)/parallelism n := step * parallelism g := newFullRandomEntryGenerator(0, n) - for i := 0; i < parallelism; i++ { + for range parallelism { gens = append(gens, newStartAtEntryGenerator(start, g)) start += step } diff --git a/pkg/storage/testing/chunk.go b/pkg/storage/testing/chunk.go index 6c625aceedd..512cf1746c6 100644 --- a/pkg/storage/testing/chunk.go +++ b/pkg/storage/testing/chunk.go @@ -76,7 +76,7 @@ func GenerateTestRandomInvalidChunk() swarm.Chunk { // Chunks by using GenerateTestRandomChunk function. func GenerateTestRandomChunks(count int) []swarm.Chunk { chunks := make([]swarm.Chunk, count) - for i := 0; i < count; i++ { + for i := range count { chunks[i] = GenerateTestRandomChunk() } return chunks diff --git a/pkg/storageincentives/events_test.go b/pkg/storageincentives/events_test.go index 0ea4547a145..f5a00d05c5b 100644 --- a/pkg/storageincentives/events_test.go +++ b/pkg/storageincentives/events_test.go @@ -41,7 +41,7 @@ func TestClose(t *testing.T) { ev.Close() - for i := 0; i < 3; i++ { + for range 3 { select { case <-done1: case <-done2: @@ -82,7 +82,7 @@ func TestPhaseCancel(t *testing.T) { ev.Publish(2) ev.Publish(3) - for i := 0; i < 2; i++ { + for range 2 { select { case <-done1: case <-done2: diff --git a/pkg/storageincentives/proof_test.go b/pkg/storageincentives/proof_test.go index de7e14f4e13..b0fcc8c8952 100644 --- a/pkg/storageincentives/proof_test.go +++ b/pkg/storageincentives/proof_test.go @@ -68,7 +68,7 @@ func TestMakeInclusionProofsRegression(t *testing.T) { // generate chunks that will be used as sample sampleChunks := make([]swarm.Chunk, 0, sampleSize) - for i := 0; i < sampleSize; i++ { + for i := range sampleSize { ch, err := cac.New(fmt.Appendf(nil, "Unstoppable data! Chunk #%d", i+1)) if err != nil { t.Fatal(err) diff --git a/pkg/storageincentives/redistributionstate_test.go b/pkg/storageincentives/redistributionstate_test.go index 063991c31d8..38c3ae8280d 100644 --- a/pkg/storageincentives/redistributionstate_test.go +++ b/pkg/storageincentives/redistributionstate_test.go @@ -199,7 +199,7 @@ func TestPurgeRoundData(t *testing.T) { hasRoundData := make([]bool, roundsCount) // Populate data at random rounds - for i := uint64(0); i < roundsCount; i++ { + for i := range uint64(roundsCount) { v := rand.Int()%2 == 0 hasRoundData[i] = v if v { @@ -210,7 +210,7 @@ func TestPurgeRoundData(t *testing.T) { // Run purge successively and assert that all data is purged up to // currentRound - purgeDataOlderThenXRounds - for i := uint64(0); i < roundsCount; i++ { + for i := range uint64(roundsCount) { state.SetCurrentEvent(0, i) state.purgeStaleRoundData() @@ -229,7 +229,7 @@ func TestPurgeRoundData(t *testing.T) { state.purgeStaleRoundData() // One more time assert that everything was purged - for i := uint64(0); i < roundsCount; i++ { + for i := range uint64(roundsCount) { assertHasDataAtRound(i, false) } } diff --git a/pkg/storageincentives/soc_mine_test.go b/pkg/storageincentives/soc_mine_test.go index 0e3a5090f47..4f3cd7812e8 100644 --- a/pkg/storageincentives/soc_mine_test.go +++ b/pkg/storageincentives/soc_mine_test.go @@ -46,7 +46,7 @@ func TestSocMine(t *testing.T) { // this constant is for a minimum reserve size of 2 million chunks with sample size of 16 // = 1.284401 * 10^71 = 1284401 + 66 0-s mstring := "1284401" - for i := 0; i < 66; i++ { + for range 66 { mstring = mstring + "0" } n, ok := new(big.Int).SetString(mstring, 10) @@ -111,7 +111,7 @@ func makeChunks(t *testing.T, signer crypto.Signer, sampleSize int, filterSOCAdd // the main loop terminating after sampleSize SOCs have been generated eg.Go(func() error { defer cancel() - for i := 0; i < sampleSize; i++ { + for i := range sampleSize { select { case sample[i] = <-sampleC: case <-ectx.Done(): @@ -125,7 +125,7 @@ func makeChunks(t *testing.T, signer crypto.Signer, sampleSize int, filterSOCAdd // loop to start mining workers count := 8 // number of parallel workers wg := sync.WaitGroup{} - for i := 0; i < count; i++ { + for i := range count { wg.Add(1) eg.Go(func() (err error) { offset := i * 4 diff --git a/pkg/storer/compact.go b/pkg/storer/compact.go index 20bf9e027e9..32a3f60dc67 100644 --- a/pkg/storer/compact.go +++ b/pkg/storer/compact.go @@ -51,7 +51,7 @@ func Compact(ctx context.Context, basePath string, opts *Options, validate bool) n := time.Now() - for shard := 0; shard < sharkyNoOfShards; shard++ { + for shard := range sharkyNoOfShards { select { case <-ctx.Done(): diff --git a/pkg/storer/compact_test.go b/pkg/storer/compact_test.go index 3fb58f22377..deb8b7412c4 100644 --- a/pkg/storer/compact_test.go +++ b/pkg/storer/compact_test.go @@ -37,14 +37,15 @@ func TestCompact(t *testing.T) { } st.StartReserveWorker(ctx, pullerMock.NewMockRateReporter(0), networkRadiusFunc(0)) - var chunks []swarm.Chunk batches := []*postage.Batch{postagetesting.MustNewBatch(), postagetesting.MustNewBatch(), postagetesting.MustNewBatch()} evictBatch := batches[1] putter := st.ReservePutter() - for b := 0; b < len(batches); b++ { - for i := uint64(0); i < 100; i++ { + chunks := make([]swarm.Chunk, 0, len(batches)*100) + + for b := range batches { + for range uint64(100) { ch := chunk.GenerateTestRandomChunk() ch = ch.WithStamp(postagetesting.MustNewBatchStamp(batches[b].ID)) chunks = append(chunks, ch) @@ -81,7 +82,7 @@ func TestCompact(t *testing.T) { } putter = st.ReservePutter() - for i := uint64(0); i < 100; i++ { + for range uint64(100) { ch := chunk.GenerateTestRandomChunk() ch = ch.WithStamp(postagetesting.MustNewBatchStamp(batches[0].ID)) chunks = append(chunks, ch) @@ -135,13 +136,14 @@ func TestCompactNoEvictions(t *testing.T) { } st.StartReserveWorker(ctx, pullerMock.NewMockRateReporter(0), networkRadiusFunc(0)) - var chunks []swarm.Chunk batches := []*postage.Batch{postagetesting.MustNewBatch(), postagetesting.MustNewBatch(), postagetesting.MustNewBatch()} putter := st.ReservePutter() - for b := 0; b < len(batches); b++ { - for i := uint64(0); i < 100; i++ { + chunks := make([]swarm.Chunk, 0, len(batches)*100) + + for b := range batches { + for range uint64(100) { ch := chunk.GenerateTestRandomChunk() ch = ch.WithStamp(postagetesting.MustNewBatchStamp(batches[b].ID)) chunks = append(chunks, ch) @@ -167,7 +169,7 @@ func TestCompactNoEvictions(t *testing.T) { } putter = st.ReservePutter() - for i := uint64(0); i < 100; i++ { + for range uint64(100) { ch := chunk.GenerateTestRandomChunk() ch = ch.WithStamp(postagetesting.MustNewBatchStamp(batches[0].ID)) chunks = append(chunks, ch) diff --git a/pkg/storer/debug_test.go b/pkg/storer/debug_test.go index 1e8149ba795..8e68b8ac972 100644 --- a/pkg/storer/debug_test.go +++ b/pkg/storer/debug_test.go @@ -157,7 +157,7 @@ func testDebugInfo(t *testing.T, newStorer func() (*storer.DB, swarm.Address, er putter := lstore.ReservePutter() - for i := 0; i < 10; i++ { + for range 10 { chunk := chunktest.GenerateTestRandomChunkAt(t, addr, 0) err := putter.Put(context.Background(), chunk) if err != nil { diff --git a/pkg/storer/export_test.go b/pkg/storer/export_test.go index 83c0bd62657..b30b5996edd 100644 --- a/pkg/storer/export_test.go +++ b/pkg/storer/export_test.go @@ -22,11 +22,11 @@ func ReplaceSharkyShardLimit(val int) { } func (db *DB) WaitForBgCacheWorkers() (unblock func()) { - for i := 0; i < defaultBgCacheWorkers; i++ { + for range defaultBgCacheWorkers { db.cacheLimiter.sem <- struct{}{} } return func() { - for i := 0; i < defaultBgCacheWorkers; i++ { + for range defaultBgCacheWorkers { <-db.cacheLimiter.sem } } diff --git a/pkg/storer/internal/cache/cache_test.go b/pkg/storer/internal/cache/cache_test.go index 79536960d94..580263cb6ac 100644 --- a/pkg/storer/internal/cache/cache_test.go +++ b/pkg/storer/internal/cache/cache_test.go @@ -211,7 +211,7 @@ func TestCache(t *testing.T) { }) t.Run("not in chunkstore returns error", func(t *testing.T) { - for i := 0; i < 5; i++ { + for range 5 { unknownChunk := chunktest.GenerateTestRandomChunk() _, err := c.Getter(st).Get(context.TODO(), unknownChunk.Address()) if !errors.Is(err, storage.ErrNotFound) { @@ -223,7 +223,7 @@ func TestCache(t *testing.T) { t.Run("not in cache doesn't affect state", func(t *testing.T) { state := c.State(st.IndexStore()) - for i := 0; i < 5; i++ { + for range 5 { extraChunk := chunktest.GenerateTestRandomChunk() err := st.Run(context.Background(), func(s transaction.Store) error { return s.ChunkStore().Put(context.TODO(), extraChunk) diff --git a/pkg/storer/internal/reserve/reserve.go b/pkg/storer/internal/reserve/reserve.go index 67a0043ccf5..e90fa27b999 100644 --- a/pkg/storer/internal/reserve/reserve.go +++ b/pkg/storer/internal/reserve/reserve.go @@ -634,7 +634,7 @@ func (r *Reserve) Reset(ctx context.Context) error { // step 4: delete binItems err = r.st.Run(context.Background(), func(s transaction.Store) error { - for i := uint8(0); i < swarm.MaxBins; i++ { + for i := range swarm.MaxBins { err := s.IndexStore().Delete(&BinItem{Bin: i}) if err != nil { return err @@ -691,7 +691,7 @@ func (r *Reserve) LastBinIDs() ([]uint64, uint64, error) { ids := make([]uint64, swarm.MaxBins) - for bin := uint8(0); bin < swarm.MaxBins; bin++ { + for bin := range swarm.MaxBins { binItem := &BinItem{Bin: bin} err := r.st.IndexStore().Get(binItem) if err != nil { diff --git a/pkg/storer/internal/reserve/reserve_test.go b/pkg/storer/internal/reserve/reserve_test.go index 4633c140385..43c2252d736 100644 --- a/pkg/storer/internal/reserve/reserve_test.go +++ b/pkg/storer/internal/reserve/reserve_test.go @@ -48,7 +48,7 @@ func TestReserve(t *testing.T) { t.Fatal(err) } - for b := 0; b < 2; b++ { + for b := range 2 { for i := 1; i < 51; i++ { ch := chunk.GenerateTestRandomChunkAt(t, baseAddr, b) err := r.Put(context.Background(), ch) @@ -101,7 +101,7 @@ func TestReserveChunkType(t *testing.T) { storedChunksCA := 0 storedChunksSO := 0 - for i := 0; i < 100; i++ { + for range 100 { ch := chunk.GenerateTestRandomChunk() if rand.Intn(2) == 0 { storedChunksCA++ @@ -552,8 +552,8 @@ func TestEvict(t *testing.T) { t.Fatal(err) } - for i := 0; i < chunksPerBatch; i++ { - for b := 0; b < 3; b++ { + for range chunksPerBatch { + for b := range 3 { ch := chunk.GenerateTestRandomChunkAt(t, baseAddr, b).WithStamp(postagetesting.MustNewBatchStamp(batches[b].ID)) chunks = append(chunks, ch) err := r.Put(context.Background(), ch) @@ -564,7 +564,7 @@ func TestEvict(t *testing.T) { } totalEvicted := 0 - for i := 0; i < 3; i++ { + for i := range 3 { evicted, err := r.EvictBatchBin(context.Background(), evictBatch.ID, math.MaxInt, uint8(i)) if err != nil { t.Fatal(err) @@ -623,9 +623,9 @@ func TestEvictSOC(t *testing.T) { batch := postagetesting.MustNewBatch() signer := getSigner(t) - var chunks []swarm.Chunk + chunks := make([]swarm.Chunk, 0, 10) - for i := 0; i < 10; i++ { + for i := range 10 { ch := soctesting.GenerateMockSocWithSigner(t, []byte{byte(i)}, signer).Chunk().WithStamp(postagetesting.MustNewFields(batch.ID, uint64(i), uint64(i))) chunks = append(chunks, ch) err := r.Put(context.Background(), ch) @@ -694,8 +694,8 @@ func TestEvictMaxCount(t *testing.T) { batch := postagetesting.MustNewBatch() - for b := 0; b < 2; b++ { - for i := 0; i < 10; i++ { + for b := range 2 { + for range 10 { ch := chunk.GenerateTestRandomChunkAt(t, baseAddr, b).WithStamp(postagetesting.MustNewBatchStamp(batch.ID)) chunks = append(chunks, ch) err := r.Put(context.Background(), ch) @@ -750,8 +750,8 @@ func TestIterate(t *testing.T) { t.Fatal(err) } - for b := 0; b < 3; b++ { - for i := 0; i < 10; i++ { + for b := range 3 { + for range 10 { ch := chunk.GenerateTestRandomChunkAt(t, baseAddr, b) err := r.Put(context.Background(), ch) if err != nil { @@ -868,7 +868,7 @@ func TestReset(t *testing.T) { total = bins * chunksPerBin ) - for b := 0; b < bins; b++ { + for b := range bins { for i := 1; i <= chunksPerBin; i++ { ch := chunk.GenerateTestRandomChunkAt(t, baseAddr, b) err := r.Put(context.Background(), ch) @@ -995,7 +995,7 @@ func TestEvictRemovesPinnedContent(t *testing.T) { batch := postagetesting.MustNewBatch() chunks := make([]swarm.Chunk, numChunks) - for i := 0; i < numChunks; i++ { + for i := range numChunks { ch := chunk.GenerateTestRandomChunkAt(t, baseAddr, 0).WithStamp(postagetesting.MustNewBatchStamp(batch.ID)) chunks[i] = ch diff --git a/pkg/storer/migration/refCntSize_test.go b/pkg/storer/migration/refCntSize_test.go index bfbc8c687e7..d5e5424283e 100644 --- a/pkg/storer/migration/refCntSize_test.go +++ b/pkg/storer/migration/refCntSize_test.go @@ -24,8 +24,8 @@ func Test_RefCntSize(t *testing.T) { store := inmemstore.New() // simulate old cacheEntryItem with some random bytes. - var oldItems []*localmigration.OldRetrievalIndexItem - for i := 0; i < 10; i++ { + oldItems := make([]*localmigration.OldRetrievalIndexItem, 0, 10) + for range 10 { entry := &localmigration.OldRetrievalIndexItem{ Address: swarm.RandAddress(t), Timestamp: uint64(rand.Int()), diff --git a/pkg/storer/migration/reserveRepair.go b/pkg/storer/migration/reserveRepair.go index 938e6b04b28..b8378d92550 100644 --- a/pkg/storer/migration/reserveRepair.go +++ b/pkg/storer/migration/reserveRepair.go @@ -84,7 +84,7 @@ func ReserveRepairer( // STEP 1 err = st.Run(context.Background(), func(s transaction.Store) error { - for i := uint8(0); i < swarm.MaxBins; i++ { + for i := range swarm.MaxBins { err := s.IndexStore().Delete(&reserve.BinItem{Bin: i}) if err != nil { return err diff --git a/pkg/storer/migration/reserveRepair_test.go b/pkg/storer/migration/reserveRepair_test.go index 884664c8787..b60f51769eb 100644 --- a/pkg/storer/migration/reserveRepair_test.go +++ b/pkg/storer/migration/reserveRepair_test.go @@ -33,15 +33,15 @@ func TestReserveRepair(t *testing.T) { var chunksPO = make([][]swarm.Chunk, 5) var chunksPerPO uint64 = 2 - for i := uint8(0); i < swarm.MaxBins; i++ { + for i := range swarm.MaxBins { err := store.Run(context.Background(), func(s transaction.Store) error { return s.IndexStore().Put(&reserve.BinItem{Bin: i, BinID: 10}) }) assert.NoError(t, err) } - for b := 0; b < 5; b++ { - for i := uint64(0); i < chunksPerPO; i++ { + for b := range 5 { + for range chunksPerPO { ch := chunktest.GenerateTestRandomChunkAt(t, baseAddr, b) stampHash, err := ch.Stamp().Hash() if err != nil { @@ -115,7 +115,7 @@ func TestReserveRepair(t *testing.T) { ) assert.NoError(t, err) - for b := 0; b < 5; b++ { + for b := range 5 { if b < 2 { if _, found := binIDs[uint8(b)]; found { t.Fatalf("bin %d should not have any binIDs", b) diff --git a/pkg/storer/migration/step_02_test.go b/pkg/storer/migration/step_02_test.go index 32cfa9abe4d..c1c741ac2fa 100644 --- a/pkg/storer/migration/step_02_test.go +++ b/pkg/storer/migration/step_02_test.go @@ -54,8 +54,8 @@ func Test_Step_02(t *testing.T) { store := internal.NewInmemStorage() // simulate old cacheEntryItem with some random bytes. - var addrs []*testEntry - for i := 0; i < 10; i++ { + addrs := make([]*testEntry, 0, 10) + for range 10 { entry := &testEntry{address: swarm.RandAddress(t)} addrs = append(addrs, entry) err := store.Run(context.Background(), func(s transaction.Store) error { diff --git a/pkg/storer/migration/step_04_test.go b/pkg/storer/migration/step_04_test.go index 758ddc7987f..fcf54225af4 100644 --- a/pkg/storer/migration/step_04_test.go +++ b/pkg/storer/migration/step_04_test.go @@ -84,7 +84,7 @@ func Test_Step_04(t *testing.T) { _, err = f.Read(buf) assert.NoError(t, err) - for i := 0; i < 10; i++ { + for i := range 10 { if i < 2 { // if the chunk is deleted, the bit is set to 1 assert.Greater(t, buf[i/8]&(1<<(i%8)), byte(0)) diff --git a/pkg/storer/reserve.go b/pkg/storer/reserve.go index 2ac9d318623..74a3b4286ce 100644 --- a/pkg/storer/reserve.go +++ b/pkg/storer/reserve.go @@ -466,9 +466,7 @@ func (db *DB) SubscribeBin(ctx context.Context, bin uint8, start uint64) (<-chan done := make(chan struct{}) errC := make(chan error, 1) - db.inFlight.Add(1) - go func() { - defer db.inFlight.Done() + db.inFlight.Go(func() { trigger, unsub := db.reserveBinEvents.Subscribe(string(bin)) defer unsub() @@ -507,7 +505,7 @@ func (db *DB) SubscribeBin(ctx context.Context, bin uint8, start uint64) (<-chan return } } - }() + }) var doneOnce sync.Once return out, func() { @@ -555,7 +553,7 @@ func neighborhoodPrefixes(base swarm.Address, radius int, suffixLength int) []sw bitCombinationsCount := int(math.Pow(2, float64(suffixLength))) bitSuffixes := make([]uint8, bitCombinationsCount) - for i := 0; i < bitCombinationsCount; i++ { + for i := range bitCombinationsCount { bitSuffixes[i] = uint8(i) } diff --git a/pkg/storer/reserve_test.go b/pkg/storer/reserve_test.go index c354f92cdbe..3126a45015c 100644 --- a/pkg/storer/reserve_test.go +++ b/pkg/storer/reserve_test.go @@ -198,8 +198,8 @@ func TestEvictBatch(t *testing.T) { putter := st.ReservePutter() - for b := 0; b < 3; b++ { - for i := uint64(0); i < chunksPerPO; i++ { + for b := range 3 { + for range chunksPerPO { ch := chunk.GenerateTestRandomChunkAt(t, baseAddr, b) ch = ch.WithStamp(postagetesting.MustNewBatchStamp(batches[b].ID)) chunks = append(chunks, ch) @@ -290,8 +290,8 @@ func TestUnreserveCap(t *testing.T) { c, unsub := storer.Events().Subscribe("reserveUnreserved") defer unsub() - for b := 0; b < 5; b++ { - for i := uint64(0); i < chunksPerPO; i++ { + for b := range 5 { + for range chunksPerPO { ch := chunk.GenerateTestRandomChunkAt(t, baseAddr, b) ch = ch.WithStamp(postagetesting.MustNewBatchStamp(batch.ID)) chunksPO[b] = append(chunksPO[b], ch) @@ -438,8 +438,8 @@ func TestRadiusManager(t *testing.T) { putter := storer.ReservePutter() - for i := 0; i < 4; i++ { - for j := 0; j < 10; j++ { + for i := range 4 { + for range 10 { ch := chunk.GenerateTestRandomChunkAt(t, baseAddr, i).WithStamp(postagetesting.MustNewBatchStamp(batch.ID)) err := putter.Put(context.Background(), ch) if err != nil { @@ -480,8 +480,8 @@ func TestSubscribeBin(t *testing.T) { putter = storer.ReservePutter() ) - for j := 0; j < 2; j++ { - for i := uint64(0); i < chunksPerPO; i++ { + for j := range 2 { + for range chunksPerPO { ch := chunk.GenerateTestRandomChunkAt(t, baseAddr, j) chunks = append(chunks, ch) err := putter.Put(context.Background(), ch) @@ -597,8 +597,8 @@ func TestSubscribeBinTrigger(t *testing.T) { ) putter := storer.ReservePutter() - for j := 0; j < 2; j++ { - for i := uint64(0); i < chunksPerPO; i++ { + for j := range 2 { + for range chunksPerPO { ch := chunk.GenerateTestRandomChunkAt(t, baseAddr, j) chunks = append(chunks, ch) err := putter.Put(context.Background(), ch) @@ -694,7 +694,7 @@ func TestNeighborhoodStats(t *testing.T) { putChunks := func(addr swarm.Address, startingRadius int, st *storer.DB) { putter := st.ReservePutter() - for i := 0; i < chunkCountPerPO; i++ { + for range chunkCountPerPO { ch := chunk.GenerateValidRandomChunkAt(t, addr, startingRadius) err := putter.Put(context.Background(), ch) if err != nil { diff --git a/pkg/storer/sample_test.go b/pkg/storer/sample_test.go index fd39882d367..49745c4b24f 100644 --- a/pkg/storer/sample_test.go +++ b/pkg/storer/sample_test.go @@ -25,8 +25,8 @@ func TestReserveSampler(t *testing.T) { randChunks := func(baseAddr swarm.Address, timeVar uint64) []swarm.Chunk { var chs []swarm.Chunk - for po := 0; po < maxPO; po++ { - for i := 0; i < chunkCountPerPO; i++ { + for po := range maxPO { + for range chunkCountPerPO { ch := chunk.GenerateValidRandomChunkAt(t, baseAddr, po).WithBatch(3, 2, false) if rand.Intn(2) == 0 { // 50% chance to wrap CAC into SOC ch = chunk.GenerateTestRandomSoChunk(t, ch) @@ -155,7 +155,7 @@ func TestReserveSamplerSisterNeighborhood(t *testing.T) { randChunks := func(baseAddr swarm.Address, startingRadius int, timeVar uint64) []swarm.Chunk { var chs []swarm.Chunk for po := startingRadius; po < maxPO; po++ { - for i := 0; i < chunkCountPerPO; i++ { + for range chunkCountPerPO { ch := chunk.GenerateValidRandomChunkAt(t, baseAddr, po).WithBatch(3, 2, false) if rand.Intn(2) == 0 { // 50% chance to wrap CAC into SOC ch = chunk.GenerateTestRandomSoChunk(t, ch) diff --git a/pkg/storer/subscribe_push.go b/pkg/storer/subscribe_push.go index bd34956b387..237b155b66c 100644 --- a/pkg/storer/subscribe_push.go +++ b/pkg/storer/subscribe_push.go @@ -23,9 +23,7 @@ func (db *DB) SubscribePush(ctx context.Context) (<-chan swarm.Chunk, func()) { stopChanOnce sync.Once ) - db.subscriptionsWG.Add(1) - go func() { - defer db.subscriptionsWG.Done() + db.subscriptionsWG.Go(func() { trigger, unsub := db.events.Subscribe(subscribePushEventKey) defer unsub() @@ -80,7 +78,7 @@ func (db *DB) SubscribePush(ctx context.Context) (<-chan swarm.Chunk, func()) { // wait for the next event } } - }() + }) stop := func() { stopChanOnce.Do(func() { diff --git a/pkg/storer/uploadstore_test.go b/pkg/storer/uploadstore_test.go index 095904ade7d..fc53e5281d6 100644 --- a/pkg/storer/uploadstore_test.go +++ b/pkg/storer/uploadstore_test.go @@ -196,7 +196,7 @@ func testUploadStore(t *testing.T, newStorer func() (*storer.DB, error)) { chunks := chunktesting.GenerateTestRandomChunks(10) for _, ch := range chunks { - for i := 0; i < 2; i++ { + for range 2 { err := session.Put(context.TODO(), ch) if err != nil { t.Fatalf("session.Put(...): unexpected error: %v", err) @@ -273,7 +273,7 @@ func testListDeleteSessions(t *testing.T, newStorer func() (*storer.DB, error)) t.Fatal(err) } - for i := 0; i < 10; i++ { + for range 10 { _, err := lstore.NewSession() if err != nil { t.Fatalf("NewSession(): unexpected error: %v", err) diff --git a/pkg/storer/validate.go b/pkg/storer/validate.go index 0220c4fb899..01f6904d02c 100644 --- a/pkg/storer/validate.go +++ b/pkg/storer/validate.go @@ -153,15 +153,13 @@ func validateWork(logger log.Logger, store storage.Store, readFn func(context.Co var wg sync.WaitGroup - for i := 0; i < 8; i++ { - wg.Add(1) - go func() { - defer wg.Done() + for range 8 { + wg.Go(func() { buf := make([]byte, swarm.SocMaxChunkSize) for item := range iteratateItemsC { validChunk(item, buf[:item.Location.Length]) } - }() + }) } count := 0 @@ -331,10 +329,8 @@ func (p *PinIntegrity) Check(ctx context.Context, logger log.Logger, pin string, iteratateItemsC := make(chan *chunkstore.RetrievalIndexItem) - for i := 0; i < 8; i++ { - wg.Add(1) - go func() { - defer wg.Done() + for range 8 { + wg.Go(func() { buf := make([]byte, swarm.SocMaxChunkSize) for item := range iteratateItemsC { if ctx.Err() != nil { @@ -344,7 +340,7 @@ func (p *PinIntegrity) Check(ctx context.Context, logger log.Logger, pin string, invalid.Add(1) } } - }() + }) } var count, micrs int64 diff --git a/pkg/swarm/proximity.go b/pkg/swarm/proximity.go index d68b9cd4795..43fefa81de2 100644 --- a/pkg/swarm/proximity.go +++ b/pkg/swarm/proximity.go @@ -28,7 +28,7 @@ func Proximity(one, other []byte) (ret uint8) { var m uint8 = 8 for i := uint8(0); i < b; i++ { oxo := one[i] ^ other[i] - for j := uint8(0); j < m; j++ { + for j := range m { if (oxo>>(7-j))&0x01 != 0 { return i*8 + j } @@ -48,7 +48,7 @@ func ExtendedProximity(one, other []byte) (ret uint8) { var m uint8 = 8 for i := uint8(0); i < b; i++ { oxo := one[i] ^ other[i] - for j := uint8(0); j < m; j++ { + for j := range m { if (oxo>>(7-j))&0x01 != 0 { return i*8 + j } diff --git a/pkg/swarm/test_helpers.go b/pkg/swarm/test_helpers.go index b043e1bb17d..e0d614ae0b1 100644 --- a/pkg/swarm/test_helpers.go +++ b/pkg/swarm/test_helpers.go @@ -55,7 +55,7 @@ func RandAddresses(tb testing.TB, count int) []Address { tb.Helper() result := make([]Address, count) - for i := 0; i < count; i++ { + for i := range count { result[i] = RandAddress(tb) } return result diff --git a/pkg/swarm/test_helpers_test.go b/pkg/swarm/test_helpers_test.go index e841fd5c6c5..11d88842681 100644 --- a/pkg/swarm/test_helpers_test.go +++ b/pkg/swarm/test_helpers_test.go @@ -29,7 +29,7 @@ func Test_RandAddressAt(t *testing.T) { hw0 := []byte{b0[0], b0[1], 0, 0} // highest words of base address hw0int := binary.BigEndian.Uint32(hw0) - for bitsInCommon := 0; bitsInCommon < 30; bitsInCommon++ { + for bitsInCommon := range 30 { addr := swarm.RandAddressAt(t, base, bitsInCommon) assertNotZeroAddress(t, addr) @@ -39,7 +39,7 @@ func Test_RandAddressAt(t *testing.T) { //bb0 is the bit mask to AND with hw0 and hw1 bb0 := uint32(0) - for i := 0; i < bitsInCommon; i++ { + for i := range bitsInCommon { bb0 |= (1 << (31 - i)) } @@ -62,7 +62,7 @@ func Test_RandAddresses(t *testing.T) { if got := len(addrs); got != count { t.Fatalf("expected %d, got %d", count, got) } - for i := 0; i < count; i++ { + for i := range count { assertNotZeroAddress(t, addrs[i]) } } diff --git a/pkg/swarm/utilities_test.go b/pkg/swarm/utilities_test.go index d3be0cac487..9fbd270513a 100644 --- a/pkg/swarm/utilities_test.go +++ b/pkg/swarm/utilities_test.go @@ -204,7 +204,7 @@ func Test_FindStampWithBatchID(t *testing.T) { func cloneAddresses(addrs []swarm.Address) []swarm.Address { result := make([]swarm.Address, len(addrs)) - for i := 0; i < len(addrs); i++ { + for i := range addrs { result[i] = addrs[i].Clone() } return result diff --git a/pkg/topology/kademlia/binprefix.go b/pkg/topology/kademlia/binprefix.go index 94e2550ea2d..4b1b119cffe 100644 --- a/pkg/topology/kademlia/binprefix.go +++ b/pkg/topology/kademlia/binprefix.go @@ -17,7 +17,7 @@ func generateCommonBinPrefixes(base swarm.Address, suffixLength int) [][]swarm.A bitCombinationsCount := int(math.Pow(2, float64(suffixLength))) bitSuffixes := make([]uint8, bitCombinationsCount) - for i := 0; i < bitCombinationsCount; i++ { + for i := range bitCombinationsCount { bitSuffixes[i] = uint8(i) } diff --git a/pkg/topology/kademlia/internal/metrics/metrics.go b/pkg/topology/kademlia/internal/metrics/metrics.go index 272dfea6d3f..706271ebcaa 100644 --- a/pkg/topology/kademlia/internal/metrics/metrics.go +++ b/pkg/topology/kademlia/internal/metrics/metrics.go @@ -297,7 +297,7 @@ func (c *Collector) Snapshot(t time.Time, addresses ...swarm.Address) map[string } if len(addresses) == 0 { - c.counters.Range(func(key, val interface{}) bool { + c.counters.Range(func(key, val any) bool { cs := val.(*Counters) snapshot[cs.peerAddress.ByteString()] = cs.snapshot(t) return true @@ -381,8 +381,8 @@ func (c *Collector) Inspect(addr swarm.Address) *Snapshot { // Flush sync the dirty in memory counters for all peers by flushing their // values to the underlying storage. func (c *Collector) Flush() error { - counters := make(map[string]interface{}) - c.counters.Range(func(key, val interface{}) bool { + counters := make(map[string]any) + c.counters.Range(func(key, val any) bool { cs := val.(*Counters) counters[cs.peerAddress.ByteString()] = val return true @@ -396,7 +396,7 @@ func (c *Collector) Flush() error { // Finalize tries to log out all ongoing peer sessions. func (c *Collector) Finalize(t time.Time, remove bool) error { - c.counters.Range(func(_, val interface{}) bool { + c.counters.Range(func(_, val any) bool { cs := val.(*Counters) PeerLogOut(t)(cs) return true @@ -407,7 +407,7 @@ func (c *Collector) Finalize(t time.Time, remove bool) error { } if remove { - c.counters.Range(func(_, val interface{}) bool { + c.counters.Range(func(_, val any) bool { cs := val.(*Counters) c.counters.Delete(cs.peerAddress.ByteString()) return true diff --git a/pkg/topology/kademlia/kademlia.go b/pkg/topology/kademlia/kademlia.go index 78d5feb0c40..e045c9658fe 100644 --- a/pkg/topology/kademlia/kademlia.go +++ b/pkg/topology/kademlia/kademlia.go @@ -533,9 +533,7 @@ func (k *Kad) manage() { balanceChan := make(chan *peerConnInfo) go k.connectionAttemptsHandler(ctx, &wg, neighbourhoodChan, balanceChan) - k.wg.Add(1) - go func() { - defer k.wg.Done() + k.wg.Go(func() { for { select { case <-k.halt: @@ -546,11 +544,9 @@ func (k *Kad) manage() { k.opt.PruneFunc(k.neighborhoodDepth()) } } - }() + }) - k.wg.Add(1) - go func() { - defer k.wg.Done() + k.wg.Go(func() { for { select { case <-k.halt: @@ -569,12 +565,10 @@ func (k *Kad) manage() { } } } - }() + }) // tell each neighbor about other neighbors periodically - k.wg.Add(1) - go func() { - defer k.wg.Done() + k.wg.Go(func() { for { select { case <-k.halt: @@ -597,7 +591,7 @@ func (k *Kad) manage() { } } } - }() + }) for { select { @@ -1041,7 +1035,7 @@ func (k *Kad) Announce(ctx context.Context, peer swarm.Address, fullnode bool) e isNeighbor := swarm.Proximity(peer.Bytes(), k.base.Bytes()) >= depth outer: - for bin := uint8(0); bin < swarm.MaxBins; bin++ { + for bin := range swarm.MaxBins { var ( connectedPeers []swarm.Address @@ -1589,7 +1583,7 @@ func randomSubset(addrs []swarm.Address, count int) ([]swarm.Address, error) { return addrs, nil } - for i := 0; i < len(addrs); i++ { + for i := range addrs { b, err := random.Int(random.Reader, big.NewInt(int64(len(addrs)))) if err != nil { return nil, err diff --git a/pkg/topology/kademlia/kademlia_test.go b/pkg/topology/kademlia/kademlia_test.go index bca195cdb10..441ac24ec49 100644 --- a/pkg/topology/kademlia/kademlia_test.go +++ b/pkg/topology/kademlia/kademlia_test.go @@ -69,7 +69,7 @@ func TestNeighborhoodDepth(t *testing.T) { testutil.CleanupCloser(t, kad) // add 2 peers in bin 8 - for i := 0; i < 2; i++ { + for range 2 { addr := swarm.RandAddressAt(t, base, 8) addOne(t, signer, kad, ab, addr) @@ -79,9 +79,9 @@ func TestNeighborhoodDepth(t *testing.T) { // depth is 0 kDepth(t, kad, 0) - var shallowPeers []swarm.Address + shallowPeers := make([]swarm.Address, 0, 2) // add two first peers (po0,po1) - for i := 0; i < 2; i++ { + for i := range 2 { addr := swarm.RandAddressAt(t, base, i) addOne(t, signer, kad, ab, addr) shallowPeers = append(shallowPeers, addr) @@ -114,8 +114,8 @@ func TestNeighborhoodDepth(t *testing.T) { // now add peers from bin 0 and expect the depth // to shift. the depth will be that of the shallowest // unsaturated bin. - for i := 0; i < 7; i++ { - for j := 0; j < 3; j++ { + for i := range 7 { + for range 3 { addr := swarm.RandAddressAt(t, base, i) addOne(t, signer, kad, ab, addr) waitConn(t, &conns) @@ -148,7 +148,7 @@ func TestNeighborhoodDepth(t *testing.T) { kDepth(t, kad, 7) // now fill bin 7 so that it is saturated, expect depth 8 - for i := 0; i < 3; i++ { + for range 3 { addr := swarm.RandAddressAt(t, base, 7) addOne(t, signer, kad, ab, addr) waitConn(t, &conns) @@ -164,7 +164,7 @@ func TestNeighborhoodDepth(t *testing.T) { var addrs []swarm.Address // fill the rest up to the bin before last and check that everything works at the edges for i := 9; i < int(swarm.MaxBins); i++ { - for j := 0; j < 4; j++ { + for range 4 { addr := swarm.RandAddressAt(t, base, i) addOne(t, signer, kad, ab, addr) waitConn(t, &conns) @@ -174,7 +174,7 @@ func TestNeighborhoodDepth(t *testing.T) { } // add a whole bunch of peers in the last bin, expect depth to stay at 31 - for i := 0; i < 15; i++ { + for range 15 { addr = swarm.RandAddressAt(t, base, int(swarm.MaxPO)) addOne(t, signer, kad, ab, addr) } @@ -187,7 +187,7 @@ func TestNeighborhoodDepth(t *testing.T) { kDepth(t, kad, 30) // empty bin 9 and expect depth 9 - for i := 0; i < 4; i++ { + for i := range 4 { removeOne(kad, addrs[i]) } kDepth(t, kad, 9) @@ -216,7 +216,7 @@ func TestNeighborhoodDepthWithReachability(t *testing.T) { kad.SetStorageRadius(0) // add 2 peers in bin 8 - for i := 0; i < 2; i++ { + for range 2 { addr := swarm.RandAddressAt(t, base, 8) addOne(t, signer, kad, ab, addr) kad.Reachable(addr, p2p.ReachabilityStatusPublic) @@ -227,9 +227,9 @@ func TestNeighborhoodDepthWithReachability(t *testing.T) { // depth is 0 kDepth(t, kad, 0) - var shallowPeers []swarm.Address + shallowPeers := make([]swarm.Address, 0, 2) // add two first peers (po0,po1) - for i := 0; i < 2; i++ { + for i := range 2 { addr := swarm.RandAddressAt(t, base, i) addOne(t, signer, kad, ab, addr) kad.Reachable(addr, p2p.ReachabilityStatusPublic) @@ -262,8 +262,8 @@ func TestNeighborhoodDepthWithReachability(t *testing.T) { // now add peers from bin 0 and expect the depth // to shift. the depth will be that of the shallowest // unsaturated bin. - for i := 0; i < 7; i++ { - for j := 0; j < 3; j++ { + for i := range 7 { + for range 3 { addr := swarm.RandAddressAt(t, base, i) addOne(t, signer, kad, ab, addr) kad.Reachable(addr, p2p.ReachabilityStatusPublic) @@ -292,7 +292,7 @@ func TestNeighborhoodDepthWithReachability(t *testing.T) { kDepth(t, kad, 7) // now fill bin 7 so that it is saturated, expect depth 8 - for i := 0; i < 3; i++ { + for range 3 { addr := swarm.RandAddressAt(t, base, 7) addOne(t, signer, kad, ab, addr) kad.Reachable(addr, p2p.ReachabilityStatusPublic) @@ -310,7 +310,7 @@ func TestNeighborhoodDepthWithReachability(t *testing.T) { var addrs []swarm.Address // fill the rest up to the bin before last and check that everything works at the edges for i := 9; i < int(swarm.MaxBins); i++ { - for j := 0; j < 4; j++ { + for range 4 { addr := swarm.RandAddressAt(t, base, i) addOne(t, signer, kad, ab, addr) kad.Reachable(addr, p2p.ReachabilityStatusPublic) @@ -321,7 +321,7 @@ func TestNeighborhoodDepthWithReachability(t *testing.T) { } // add a whole bunch of peers in the last bin, expect depth to stay at 31 - for i := 0; i < 15; i++ { + for range 15 { addr = swarm.RandAddressAt(t, base, int(swarm.MaxPO)) addOne(t, signer, kad, ab, addr) kad.Reachable(addr, p2p.ReachabilityStatusPublic) @@ -335,7 +335,7 @@ func TestNeighborhoodDepthWithReachability(t *testing.T) { kDepth(t, kad, 30) // empty bin 9 and expect depth 9 - for i := 0; i < 4; i++ { + for i := range 4 { removeOne(kad, addrs[i]) } kDepth(t, kad, 9) @@ -365,7 +365,7 @@ func TestManage(t *testing.T) { kad.SetStorageRadius(0) // first, we add peers to bin 0 - for i := 0; i < saturation; i++ { + for range saturation { addr := swarm.RandAddressAt(t, base, 0) addOne(t, signer, kad, ab, addr) } @@ -373,7 +373,7 @@ func TestManage(t *testing.T) { waitCounter(t, &conns, int32(saturation)) // next, we add peers to the next bin - for i := 0; i < saturation; i++ { + for range saturation { addr := swarm.RandAddressAt(t, base, 1) addOne(t, signer, kad, ab, addr) } @@ -383,7 +383,7 @@ func TestManage(t *testing.T) { kad.SetStorageRadius(1) // here, we attempt to add to bin 0, but bin is saturated, so no new peers should connect to it - for i := 0; i < saturation; i++ { + for range saturation { addr := swarm.RandAddressAt(t, base, 0) addOne(t, signer, kad, ab, addr) } @@ -466,8 +466,8 @@ func TestBinSaturation(t *testing.T) { // add two peers in a few bins to generate some depth >= 0, this will // make the next iteration result in binSaturated==true, causing no new // connections to be made - for i := 0; i < 5; i++ { - for j := 0; j < 2; j++ { + for i := range 5 { + for range 2 { addr := swarm.RandAddressAt(t, base, i) addOne(t, signer, kad, ab, addr) } @@ -479,7 +479,7 @@ func TestBinSaturation(t *testing.T) { // add one more peer in each bin shallower than depth and // expect no connections due to saturation. if we add a peer within // depth, the short circuit will be hit and we will connect to the peer - for i := 0; i < 3; i++ { + for i := range 3 { addr := swarm.RandAddressAt(t, base, i) addOne(t, signer, kad, ab, addr) } @@ -514,8 +514,8 @@ func TestOversaturation(t *testing.T) { testutil.CleanupCloser(t, kad) // Add maximum accepted number of peers up until bin 5 without problems - for i := 0; i < 6; i++ { - for j := 0; j < kademlia.DefaultOverSaturationPeers; j++ { + for i := range 6 { + for range kademlia.DefaultOverSaturationPeers { addr := swarm.RandAddressAt(t, base, i) // if error is not nil as specified, connectOne goes fatal connectOne(t, signer, kad, ab, addr, nil) @@ -527,9 +527,9 @@ func TestOversaturation(t *testing.T) { // see depth is 5 kDepth(t, kad, 5) - for k := 0; k < 5; k++ { + for k := range 5 { // no further connections can be made - for l := 0; l < 3; l++ { + for range 3 { addr := swarm.RandAddressAt(t, base, k) // if error is not as specified, connectOne goes fatal connectOne(t, signer, kad, ab, addr, topology.ErrOversaturated) @@ -566,8 +566,8 @@ func TestOversaturationBootnode(t *testing.T) { testutil.CleanupCloser(t, kad) // Add maximum accepted number of peers up until bin 5 without problems - for i := 0; i < 6; i++ { - for j := 0; j < overSaturationPeers; j++ { + for i := range 6 { + for range overSaturationPeers { addr := swarm.RandAddressAt(t, base, i) // if error is not nil as specified, connectOne goes fatal connectOne(t, signer, kad, ab, addr, nil) @@ -579,9 +579,9 @@ func TestOversaturationBootnode(t *testing.T) { // see depth is 5 kDepth(t, kad, 5) - for k := 0; k < 5; k++ { + for k := range 5 { // further connections should succeed outside of depth - for l := 0; l < 3; l++ { + for range 3 { addr := swarm.RandAddressAt(t, base, k) // if error is not as specified, connectOne goes fatal connectOne(t, signer, kad, ab, addr, nil) @@ -595,7 +595,7 @@ func TestOversaturationBootnode(t *testing.T) { } // see we can still add / not limiting more peers in neighborhood depth - for m := 0; m < 12; m++ { + for range 12 { addr := swarm.RandAddressAt(t, base, 5) // if error is not nil as specified, connectOne goes fatal connectOne(t, signer, kad, ab, addr, nil) @@ -624,8 +624,8 @@ func TestBootnodeMaxConnections(t *testing.T) { testutil.CleanupCloser(t, kad) // Add maximum accepted number of peers up until bin 5 without problems - for i := 0; i < 6; i++ { - for j := 0; j < bootnodeOverSaturationPeers; j++ { + for i := range 6 { + for range bootnodeOverSaturationPeers { addr := swarm.RandAddressAt(t, base, i) // if error is not nil as specified, connectOne goes fatal connectOne(t, signer, kad, ab, addr, nil) @@ -640,9 +640,9 @@ func TestBootnodeMaxConnections(t *testing.T) { depth := 5 outSideDepthPeers := 5 - for k := 0; k < depth; k++ { + for k := range depth { // further connections should succeed outside of depth - for l := 0; l < outSideDepthPeers; l++ { + for range outSideDepthPeers { addr := swarm.RandAddressAt(t, base, k) // if error is not as specified, connectOne goes fatal connectOne(t, signer, kad, ab, addr, nil) @@ -1109,7 +1109,7 @@ func TestKademlia_SubscribeTopologyChange(t *testing.T) { c2, u2 := kad.SubscribeTopologyChange() defer u2() - for i := 0; i < 4; i++ { + for i := range 4 { addr := swarm.RandAddressAt(t, base, i) addOne(t, sg, kad, ab, addr) } @@ -1129,14 +1129,14 @@ func TestKademlia_SubscribeTopologyChange(t *testing.T) { c, u := kad.SubscribeTopologyChange() defer u() - for i := 0; i < 4; i++ { + for i := range 4 { addr := swarm.RandAddressAt(t, base, i) addOne(t, sg, kad, ab, addr) } testSignal(t, c) - for i := 0; i < 4; i++ { + for i := range 4 { addr := swarm.RandAddressAt(t, base, i) addOne(t, sg, kad, ab, addr) } @@ -1210,9 +1210,9 @@ func getBinPopulation(bins *topology.KadBins, po uint8) uint64 { func TestStart(t *testing.T) { t.Parallel() - var bootnodes []ma.Multiaddr - var bootnodesOverlays []swarm.Address - for i := 0; i < 10; i++ { + bootnodes := make([]ma.Multiaddr, 0, 10) + bootnodesOverlays := make([]swarm.Address, 0, 10) + for range 10 { overlay := swarm.RandAddress(t) multiaddr, err := ma.NewMultiaddr(underlayBase + overlay.String()) @@ -1231,7 +1231,7 @@ func TestStart(t *testing.T) { var conns, failedConns int32 // how many connect calls were made to the p2p mock _, kad, ab, _, signer := newTestKademlia(t, &conns, &failedConns, kademlia.Options{Bootnodes: bootnodes}) - for i := 0; i < 3; i++ { + for range 3 { peer := swarm.RandAddress(t) multiaddr, err := ma.NewMultiaddr(underlayBase + peer.String()) if err != nil { @@ -1324,7 +1324,7 @@ func TestOutofDepthPrune(t *testing.T) { testutil.CleanupCloser(t, kad) // bin 0,1 balanced, rest not - for i := 0; i < 6; i++ { + for i := range 6 { var peers []swarm.Address if i < 2 { peers = mineBin(t, base, i, 20, true) @@ -1350,7 +1350,7 @@ func TestOutofDepthPrune(t *testing.T) { // check that no pruning has happened bins := binSizes(kad) - for i := 0; i < 6; i++ { + for i := range 6 { if bins[i] <= overSaturationPeers { t.Fatalf("bin %d, got %d, want more than %d", i, bins[i], overSaturationPeers) } @@ -1375,7 +1375,7 @@ func TestOutofDepthPrune(t *testing.T) { // check bins have been pruned bins = binSizes(kad) - for i := uint8(0); i < 5; i++ { + for i := range uint8(5) { if bins[i] != overSaturationPeers { t.Fatalf("bin %d, got %d, want %d", i, bins[i], overSaturationPeers) } @@ -1426,7 +1426,7 @@ func TestPruneExcludeOps(t *testing.T) { testutil.CleanupCloser(t, kad) // bin 0,1 balanced, rest not - for i := 0; i < 6; i++ { + for i := range 6 { var peers []swarm.Address if i < 2 { peers = mineBin(t, base, i, perBin, true) @@ -1441,7 +1441,7 @@ func TestPruneExcludeOps(t *testing.T) { kad.Reachable(peers[i], p2p.ReachabilityStatusPublic) } } - for i := 0; i < 4; i++ { + for range 4 { } time.Sleep(time.Millisecond * 10) kDepth(t, kad, i) @@ -1459,7 +1459,7 @@ func TestPruneExcludeOps(t *testing.T) { // check that no pruning has happened bins := binSizes(kad) - for i := 0; i < 6; i++ { + for i := range 6 { if bins[i] <= overSaturationPeers { t.Fatalf("bin %d, got %d, want more than %d", i, bins[i], overSaturationPeers) } @@ -1484,7 +1484,7 @@ func TestPruneExcludeOps(t *testing.T) { // check bins have NOT been pruned because the peer count func excluded unreachable peers bins = binSizes(kad) - for i := uint8(0); i < 5; i++ { + for i := range uint8(5) { if bins[i] != perBin { t.Fatalf("bin %d, got %d, want %d", i, bins[i], perBin) } @@ -1501,7 +1501,7 @@ func TestBootnodeProtectedNodes(t *testing.T) { // create base and protected nodes addresses base := swarm.RandAddress(t) protected := make([]swarm.Address, 6) - for i := 0; i < 6; i++ { + for i := range 6 { addr := swarm.RandAddressAt(t, base, i) protected[i] = addr } @@ -1526,8 +1526,8 @@ func TestBootnodeProtectedNodes(t *testing.T) { testutil.CleanupCloser(t, kad) // Add maximum accepted number of peers up until bin 5 without problems - for i := 0; i < 6; i++ { - for j := 0; j < overSaturationPeers; j++ { + for i := range 6 { + for range overSaturationPeers { // if error is not nil as specified, connectOne goes fatal connectOne(t, signer, kad, ab, protected[i], nil) } @@ -1538,7 +1538,7 @@ func TestBootnodeProtectedNodes(t *testing.T) { // see depth is 5 kDepth(t, kad, 5) - for k := 0; k < 5; k++ { + for k := range 5 { // further connections should succeed outside of depth addr := swarm.RandAddressAt(t, base, k) // if error is not as specified, connectOne goes fatal @@ -1549,14 +1549,14 @@ func TestBootnodeProtectedNodes(t *testing.T) { // ensure protected node was not kicked out and we have more than oversaturation // amount sizes := binSizes(kad) - for k := 0; k < 5; k++ { + for k := range 5 { if sizes[k] != 2 { t.Fatalf("invalid bin size expected 2 found %d", sizes[k]) } } - for k := 0; k < 5; k++ { + for k := range 5 { // further connections should succeed outside of depth - for l := 0; l < 3; l++ { + for range 3 { addr := swarm.RandAddressAt(t, base, k) // if error is not as specified, connectOne goes fatal connectOne(t, signer, kad, ab, addr, nil) @@ -1567,7 +1567,7 @@ func TestBootnodeProtectedNodes(t *testing.T) { // ensure unprotected nodes are kicked out to make room for new peers and protected // nodes are still present sizes = binSizes(kad) - for k := 0; k < 5; k++ { + for k := range 5 { if sizes[k] != 2 { t.Fatalf("invalid bin size expected 2 found %d", sizes[k]) } @@ -1691,8 +1691,8 @@ func TestAnnounceNeighborhoodToNeighbor(t *testing.T) { testutil.CleanupCloser(t, kad) // add some peers - for bin := 0; bin < 2; bin++ { - for i := 0; i < 4; i++ { + for bin := range 2 { + for range 4 { addr := swarm.RandAddressAt(t, base, bin) addOne(t, signer, kad, ab, addr) waitCounter(t, &conns, 1) @@ -1703,7 +1703,7 @@ func TestAnnounceNeighborhoodToNeighbor(t *testing.T) { kDepth(t, kad, 1) // add many more neighbors - for i := 0; i < 10; i++ { + for range 10 { addr := swarm.RandAddressAt(t, base, 2) addOne(t, signer, kad, ab, addr) waitCounter(t, &conns, 1) @@ -1740,8 +1740,8 @@ func TestIteratorOpts(t *testing.T) { } testutil.CleanupCloser(t, kad) - for i := 0; i < 6; i++ { - for j := 0; j < 4; j++ { + for i := range 6 { + for range 4 { addr := swarm.RandAddressAt(t, base, i) // if error is not nil as specified, connectOne goes fatal connectOne(t, signer, kad, ab, addr, nil) @@ -1903,7 +1903,7 @@ func mineBin(t *testing.T, base swarm.Address, bin, count int, isBalanced bool) t.Fatal("peersCount must be greater than 8 for balanced bins") } - for i := 0; i < count; i++ { + for i := range count { rndAddrs[i] = swarm.RandAddressAt(t, base, bin) } diff --git a/pkg/topology/pslice/pslice_test.go b/pkg/topology/pslice/pslice_test.go index c7b540450b4..07eadaafdd8 100644 --- a/pkg/topology/pslice/pslice_test.go +++ b/pkg/topology/pslice/pslice_test.go @@ -24,8 +24,8 @@ func TestShallowestEmpty(t *testing.T) { peers = make([][]swarm.Address, 16) ) - for i := 0; i < 16; i++ { - for j := 0; j < 3; j++ { + for i := range 16 { + for range 3 { a := swarm.RandAddressAt(t, base, i) peers[i] = append(peers[i], a) } @@ -221,7 +221,7 @@ func TestIterators(t *testing.T) { ps := pslice.New(4, base) peers := make([]swarm.Address, 4) - for i := 0; i < 4; i++ { + for i := range 4 { peers[i] = swarm.RandAddressAt(t, base, i) } @@ -286,7 +286,7 @@ func TestBinPeers(t *testing.T) { // prepare slice ps := pslice.New(len(tc.peersCount), base) for bin, peersCount := range tc.peersCount { - for i := 0; i < peersCount; i++ { + for range peersCount { peer := swarm.RandAddressAt(t, base, bin) binPeers[bin] = append(binPeers[bin], peer) ps.Add(peer) @@ -342,8 +342,8 @@ func TestIteratorsJumpStop(t *testing.T) { ps := pslice.New(4, base) peers := make([]swarm.Address, 0, 12) - for i := 0; i < 4; i++ { - for ii := 0; ii < 3; ii++ { + for i := range 4 { + for range 3 { a := swarm.RandAddressAt(t, base, i) peers = append(peers, a) ps.Add(a) diff --git a/pkg/tracing/tracing_test.go b/pkg/tracing/tracing_test.go index 9c0baa2f685..c6acb330b18 100644 --- a/pkg/tracing/tracing_test.go +++ b/pkg/tracing/tracing_test.go @@ -144,7 +144,7 @@ func TestStartSpanFromContext_logger(t *testing.T) { wantTraceID := span.Context().(jaeger.SpanContext).TraceID() logger.Info("msg") - data := make(map[string]interface{}) + data := make(map[string]any) if err := json.Unmarshal(buf.Bytes(), &data); err != nil { t.Fatalf("unexpected error: %v", err) } @@ -192,7 +192,7 @@ func TestNewLoggerWithTraceID(t *testing.T) { wantTraceID := span.Context().(jaeger.SpanContext).TraceID() logger.Info("msg") - data := make(map[string]interface{}) + data := make(map[string]any) if err := json.Unmarshal(buf.Bytes(), &data); err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/pkg/transaction/event.go b/pkg/transaction/event.go index 8eb2001ebf7..587b3d86427 100644 --- a/pkg/transaction/event.go +++ b/pkg/transaction/event.go @@ -18,7 +18,7 @@ var ( ) // ParseEvent will parse the specified abi event from the given log -func ParseEvent(a *abi.ABI, eventName string, c interface{}, e types.Log) error { +func ParseEvent(a *abi.ABI, eventName string, c any, e types.Log) error { if len(e.Topics) == 0 { return ErrNoTopic } @@ -37,7 +37,7 @@ func ParseEvent(a *abi.ABI, eventName string, c interface{}, e types.Log) error } // FindSingleEvent will find the first event of the given kind. -func FindSingleEvent(abi *abi.ABI, receipt *types.Receipt, contractAddress common.Address, event abi.Event, out interface{}) error { +func FindSingleEvent(abi *abi.ABI, receipt *types.Receipt, contractAddress common.Address, event abi.Event, out any) error { if receipt.Status != 1 { return ErrTransactionReverted } diff --git a/pkg/transaction/mock/transaction.go b/pkg/transaction/mock/transaction.go index d987fd52249..072f47cf8f2 100644 --- a/pkg/transaction/mock/transaction.go +++ b/pkg/transaction/mock/transaction.go @@ -171,10 +171,10 @@ type Call struct { to common.Address result []byte method string - params []interface{} + params []any } -func ABICall(abi *abi.ABI, to common.Address, result []byte, method string, params ...interface{}) Call { +func ABICall(abi *abi.ABI, to common.Address, result []byte, method string, params ...any) Call { return Call{ to: to, abi: abi, @@ -216,11 +216,11 @@ func WithABICallSequence(calls ...Call) Option { }) } -func WithABICall(abi *abi.ABI, to common.Address, result []byte, method string, params ...interface{}) Option { +func WithABICall(abi *abi.ABI, to common.Address, result []byte, method string, params ...any) Option { return WithABICallSequence(ABICall(abi, to, result, method, params...)) } -func WithABISend(abi *abi.ABI, txHash common.Hash, expectedAddress common.Address, expectedValue *big.Int, method string, params ...interface{}) Option { +func WithABISend(abi *abi.ABI, txHash common.Hash, expectedAddress common.Address, expectedValue *big.Int, method string, params ...any) Option { return optionFunc(func(s *transactionServiceMock) { s.send = func(ctx context.Context, request *transaction.TxRequest, boost int) (common.Hash, error) { data, err := abi.Pack(method, params...) diff --git a/pkg/transaction/transaction.go b/pkg/transaction/transaction.go index 2f2405dd432..bd63d7cfce1 100644 --- a/pkg/transaction/transaction.go +++ b/pkg/transaction/transaction.go @@ -15,6 +15,7 @@ import ( "time" "context" + "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" @@ -223,9 +224,7 @@ func (t *transactionService) Send(ctx context.Context, request *TxRequest, boost } func (t *transactionService) waitForPendingTx(txHash common.Hash) { - t.wg.Add(1) - go func() { - defer t.wg.Done() + t.wg.Go(func() { switch _, err := t.WaitForReceipt(t.ctx, txHash); err { case nil: t.logger.Info("pending transaction confirmed", "tx", txHash) @@ -240,7 +239,7 @@ func (t *transactionService) waitForPendingTx(txHash common.Hash) { t.logger.Error(err, "waiting for pending transaction failed", "tx", txHash) } } - }() + }) } func (t *transactionService) Call(ctx context.Context, request *TxRequest) ([]byte, error) { @@ -597,9 +596,9 @@ func (t *transactionService) UnwrapABIError(ctx context.Context, req *TxRequest, continue } - values, ok := data.([]interface{}) + values, ok := data.([]any) if !ok { - values = make([]interface{}, len(abiError.Inputs)) + values = make([]any, len(abiError.Inputs)) for i := range values { values[i] = "?" } diff --git a/pkg/transaction/transaction_test.go b/pkg/transaction/transaction_test.go index 079e341dd27..26c3f81111d 100644 --- a/pkg/transaction/transaction_test.go +++ b/pkg/transaction/transaction_test.go @@ -852,9 +852,9 @@ type rpcAPIError struct { err string } -func (e *rpcAPIError) ErrorCode() int { return e.code } -func (e *rpcAPIError) Error() string { return e.msg } -func (e *rpcAPIError) ErrorData() interface{} { return e.err } +func (e *rpcAPIError) ErrorCode() int { return e.code } +func (e *rpcAPIError) Error() string { return e.msg } +func (e *rpcAPIError) ErrorData() any { return e.err } var _ rpc.DataError = (*rpcAPIError)(nil) diff --git a/pkg/util/testutil/pseudorand/reader_test.go b/pkg/util/testutil/pseudorand/reader_test.go index 2abd6aaf382..90f5abe11ae 100644 --- a/pkg/util/testutil/pseudorand/reader_test.go +++ b/pkg/util/testutil/pseudorand/reader_test.go @@ -97,7 +97,7 @@ func TestReader(t *testing.T) { } }) t.Run("seek and match", func(t *testing.T) { - for i := 0; i < 20; i++ { + for range 20 { off := rand.Intn(size) n := rand.Intn(size - off) t.Run(fmt.Sprintf("off=%d n=%d", off, n), func(t *testing.T) { From 0cc0e519116585df73a5cdb8ccda63b29c6863ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ljubi=C5=A1a=20Ga=C4=8Devi=C4=87?= <35105035+gacevicljubisa@users.noreply.github.com> Date: Tue, 14 Oct 2025 16:04:30 +0200 Subject: [PATCH 33/96] chore: bump toolchain to go1.25.2 and images debian:12.12-slim (#5248) --- Dockerfile | 2 +- Dockerfile.dev | 2 +- go.mod | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 01f7f2bad89..26e9920166f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ COPY . ./ RUN make binary -FROM debian:12.10-slim +FROM debian:12.12-slim ENV DEBIAN_FRONTEND=noninteractive diff --git a/Dockerfile.dev b/Dockerfile.dev index 3cfae361aea..551a6208bb8 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -13,7 +13,7 @@ RUN make binary \ REACHABILITY_OVERRIDE_PUBLIC=${REACHABILITY_OVERRIDE_PUBLIC} \ BATCHFACTOR_OVERRIDE_PUBLIC=${BATCHFACTOR_OVERRIDE_PUBLIC} -FROM debian:12.10-slim +FROM debian:12.12-slim ENV DEBIAN_FRONTEND=noninteractive diff --git a/go.mod b/go.mod index eccad953b87..5e74fa8aed6 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module github.com/ethersphere/bee/v2 go 1.25 -toolchain go1.25.1 +toolchain go1.25.2 require ( contrib.go.opencensus.io/exporter/prometheus v0.4.2 From 64ea8f6c6d3bd98acb3d444c74dad82772b5c4ab Mon Sep 17 00:00:00 2001 From: Akrem Chabchoub <121046693+akrem-chabchoub@users.noreply.github.com> Date: Mon, 20 Oct 2025 12:35:56 +0100 Subject: [PATCH 34/96] docs(tags): remove body from `POST /tags` (#5246) --- openapi/Swarm.yaml | 6 ------ openapi/SwarmCommon.yaml | 6 ------ 2 files changed, 12 deletions(-) diff --git a/openapi/Swarm.yaml b/openapi/Swarm.yaml index 48916c54af1..7f48e32d329 100644 --- a/openapi/Swarm.yaml +++ b/openapi/Swarm.yaml @@ -565,12 +565,6 @@ paths: tags: - Tag description: Tags can be thought of as upload sessions which can be tracked using the tags endpoint. It will keep track of the chunks that are uploaded as part of the tag and will push them out to the network once a done split is called on the Tag. This happens internally if you use the `Swarm-Deferred-Upload` header. - requestBody: - required: true - content: - application/json: - schema: - $ref: "SwarmCommon.yaml#/components/schemas/NewTagRequest" responses: "201": description: New Tag Info diff --git a/openapi/SwarmCommon.yaml b/openapi/SwarmCommon.yaml index 3d08175f01c..bf15dbf205e 100644 --- a/openapi/SwarmCommon.yaml +++ b/openapi/SwarmCommon.yaml @@ -295,12 +295,6 @@ components: MultiAddress: type: string - NewTagRequest: - type: object - properties: - address: - $ref: "#/components/schemas/SwarmAddress" - NewTagResponse: type: object properties: From 23ef082358590a2ac466f98c253ea219021a12ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ljubi=C5=A1a=20Ga=C4=8Devi=C4=87?= <35105035+gacevicljubisa@users.noreply.github.com> Date: Mon, 27 Oct 2025 15:02:22 +0100 Subject: [PATCH 35/96] feat(docker): add debugging and networking tools to dev container (#5255) --- Dockerfile.dev | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Dockerfile.dev b/Dockerfile.dev index 551a6208bb8..60f6f175cb3 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -18,13 +18,19 @@ FROM debian:12.12-slim ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y --no-install-recommends \ - ca-certificates; \ + ca-certificates \ + iputils-ping \ + netcat-openbsd \ + telnet \ + curl \ + wget \ + jq \ + net-tools; \ apt-get clean; \ rm -rf /var/lib/apt/lists/*; \ groupadd -r bee --gid 999; \ useradd -r -g bee --uid 999 --no-log-init -m bee; -# make sure mounted volumes have correct permissions RUN mkdir -p /home/bee/.bee && chown 999:999 /home/bee/.bee COPY --from=build /src/dist/bee /usr/local/bin/bee From bf267453b447ba7ff3d33e00625624470383f4a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ljubi=C5=A1a=20Ga=C4=8Devi=C4=87?= <35105035+gacevicljubisa@users.noreply.github.com> Date: Mon, 27 Oct 2025 15:02:44 +0100 Subject: [PATCH 36/96] fix(transaction): use EstimateGasAtBlock instead of EstimateGas (#5252) --- pkg/node/chain.go | 4 +-- pkg/transaction/backend/backend.go | 2 +- pkg/transaction/backendmock/backend.go | 12 ++++---- pkg/transaction/backendsimulation/backend.go | 2 +- pkg/transaction/transaction.go | 5 ++-- pkg/transaction/transaction_test.go | 30 ++++++++++---------- pkg/transaction/wrapped/wrapped.go | 5 ++-- 7 files changed, 29 insertions(+), 31 deletions(-) diff --git a/pkg/node/chain.go b/pkg/node/chain.go index c1f00dd61bc..e6fc2c429ec 100644 --- a/pkg/node/chain.go +++ b/pkg/node/chain.go @@ -372,8 +372,8 @@ func (m noOpChainBackend) SuggestGasTipCap(context.Context) (*big.Int, error) { panic("chain no op: SuggestGasTipCap") } -func (m noOpChainBackend) EstimateGas(context.Context, ethereum.CallMsg) (uint64, error) { - panic("chain no op: EstimateGas") +func (m noOpChainBackend) EstimateGasAtBlock(context.Context, ethereum.CallMsg, *big.Int) (uint64, error) { + panic("chain no op: EstimateGasAtBlock") } func (m noOpChainBackend) SendTransaction(context.Context, *types.Transaction) error { diff --git a/pkg/transaction/backend/backend.go b/pkg/transaction/backend/backend.go index 67323d14e45..827e7f8176a 100644 --- a/pkg/transaction/backend/backend.go +++ b/pkg/transaction/backend/backend.go @@ -20,7 +20,7 @@ type Geth interface { CallContract(ctx context.Context, msg ethereum.CallMsg, blockNumber *big.Int) ([]byte, error) ChainID(ctx context.Context) (*big.Int, error) Close() - EstimateGas(ctx context.Context, msg ethereum.CallMsg) (uint64, error) + EstimateGasAtBlock(ctx context.Context, msg ethereum.CallMsg, blockNumber *big.Int) (uint64, error) FilterLogs(ctx context.Context, q ethereum.FilterQuery) ([]types.Log, error) HeaderByNumber(ctx context.Context, number *big.Int) (*types.Header, error) NonceAt(ctx context.Context, account common.Address, blockNumber *big.Int) (uint64, error) diff --git a/pkg/transaction/backendmock/backend.go b/pkg/transaction/backendmock/backend.go index 25166a898e0..e07095fd352 100644 --- a/pkg/transaction/backendmock/backend.go +++ b/pkg/transaction/backendmock/backend.go @@ -20,7 +20,7 @@ type backendMock struct { sendTransaction func(ctx context.Context, tx *types.Transaction) error suggestedFeeAndTip func(ctx context.Context, gasPrice *big.Int, boostPercent int) (*big.Int, *big.Int, error) suggestGasTipCap func(ctx context.Context) (*big.Int, error) - estimateGas func(ctx context.Context, call ethereum.CallMsg) (gas uint64, err error) + estimateGasAtBlock func(ctx context.Context, msg ethereum.CallMsg, blockNumber *big.Int) (gas uint64, err error) transactionReceipt func(ctx context.Context, txHash common.Hash) (*types.Receipt, error) pendingNonceAt func(ctx context.Context, account common.Address) (uint64, error) transactionByHash func(ctx context.Context, hash common.Hash) (tx *types.Transaction, isPending bool, err error) @@ -51,9 +51,9 @@ func (m *backendMock) SuggestedFeeAndTip(ctx context.Context, gasPrice *big.Int, return nil, nil, errors.New("not implemented") } -func (m *backendMock) EstimateGas(ctx context.Context, call ethereum.CallMsg) (gas uint64, err error) { - if m.estimateGas != nil { - return m.estimateGas(ctx, call) +func (m *backendMock) EstimateGasAtBlock(ctx context.Context, msg ethereum.CallMsg, blockNumber *big.Int) (uint64, error) { + if m.estimateGasAtBlock != nil { + return m.estimateGasAtBlock(ctx, msg, blockNumber) } return 0, errors.New("not implemented") } @@ -171,9 +171,9 @@ func WithSuggestGasTipCapFunc(f func(ctx context.Context) (*big.Int, error)) Opt }) } -func WithEstimateGasFunc(f func(ctx context.Context, call ethereum.CallMsg) (gas uint64, err error)) Option { +func WithEstimateGasAtBlockFunc(f func(ctx context.Context, msg ethereum.CallMsg, blockNumber *big.Int) (gas uint64, err error)) Option { return optionFunc(func(s *backendMock) { - s.estimateGas = f + s.estimateGasAtBlock = f }) } diff --git a/pkg/transaction/backendsimulation/backend.go b/pkg/transaction/backendsimulation/backend.go index 47480fd3994..eacddc44f0f 100644 --- a/pkg/transaction/backendsimulation/backend.go +++ b/pkg/transaction/backendsimulation/backend.go @@ -95,7 +95,7 @@ func (m *simulatedBackend) SuggestedFeeAndTip(ctx context.Context, gasPrice *big return nil, nil, errors.New("not implemented") } -func (m *simulatedBackend) EstimateGas(ctx context.Context, call ethereum.CallMsg) (gas uint64, err error) { +func (m *simulatedBackend) EstimateGasAtBlock(ctx context.Context, msg ethereum.CallMsg, blockNumber *big.Int) (uint64, error) { return 0, errors.New("not implemented") } diff --git a/pkg/transaction/transaction.go b/pkg/transaction/transaction.go index bd63d7cfce1..307d1c46261 100644 --- a/pkg/transaction/transaction.go +++ b/pkg/transaction/transaction.go @@ -31,7 +31,6 @@ import ( const loggerName = "transaction" const ( - noncePrefix = "transaction_nonce_" storedTransactionPrefix = "transaction_stored_" pendingTransactionPrefix = "transaction_pending_" ) @@ -275,11 +274,11 @@ func (t *transactionService) StoredTransaction(txHash common.Hash) (*StoredTrans func (t *transactionService) prepareTransaction(ctx context.Context, request *TxRequest, nonce uint64, boostPercent int) (tx *types.Transaction, err error) { var gasLimit uint64 if request.GasLimit == 0 { - gasLimit, err = t.backend.EstimateGas(ctx, ethereum.CallMsg{ + gasLimit, err = t.backend.EstimateGasAtBlock(ctx, ethereum.CallMsg{ From: t.sender, To: request.To, Data: request.Data, - }) + }, nil) // nil for latest block if err != nil { t.logger.Debug("estimate gas failed", "error", err) gasLimit = request.MinEstimatedGasLimit diff --git a/pkg/transaction/transaction_test.go b/pkg/transaction/transaction_test.go index 26c3f81111d..e9f4e385c32 100644 --- a/pkg/transaction/transaction_test.go +++ b/pkg/transaction/transaction_test.go @@ -151,11 +151,11 @@ func TestTransactionSend(t *testing.T) { } return nil }), - backendmock.WithEstimateGasFunc(func(ctx context.Context, call ethereum.CallMsg) (gas uint64, err error) { - if !bytes.Equal(call.To.Bytes(), recipient.Bytes()) { - t.Fatalf("estimating with wrong recipient. wanted %x, got %x", recipient, call.To) + backendmock.WithEstimateGasAtBlockFunc(func(ctx context.Context, msg ethereum.CallMsg, blockNumber *big.Int) (gas uint64, err error) { + if !bytes.Equal(msg.To.Bytes(), recipient.Bytes()) { + t.Fatalf("estimating with wrong recipient. wanted %x, got %x", recipient, msg.To) } - if !bytes.Equal(call.Data, txData) { + if !bytes.Equal(msg.Data, txData) { t.Fatal("estimating with wrong data") } return estimatedGasLimit, nil @@ -234,7 +234,7 @@ func TestTransactionSend(t *testing.T) { } return nil }), - backendmock.WithEstimateGasFunc(func(ctx context.Context, call ethereum.CallMsg) (gas uint64, err error) { + backendmock.WithEstimateGasAtBlockFunc(func(ctx context.Context, msg ethereum.CallMsg, blockNumber *big.Int) (gas uint64, err error) { return 0, errors.New("estimate failure") }), backendmock.WithPendingNonceAtFunc(func(ctx context.Context, account common.Address) (uint64, error) { @@ -314,11 +314,11 @@ func TestTransactionSend(t *testing.T) { } return nil }), - backendmock.WithEstimateGasFunc(func(ctx context.Context, call ethereum.CallMsg) (gas uint64, err error) { - if !bytes.Equal(call.To.Bytes(), recipient.Bytes()) { - t.Fatalf("estimating with wrong recipient. wanted %x, got %x", recipient, call.To) + backendmock.WithEstimateGasAtBlockFunc(func(ctx context.Context, msg ethereum.CallMsg, blockNumber *big.Int) (gas uint64, err error) { + if !bytes.Equal(msg.To.Bytes(), recipient.Bytes()) { + t.Fatalf("estimating with wrong recipient. wanted %x, got %x", recipient, msg.To) } - if !bytes.Equal(call.Data, txData) { + if !bytes.Equal(msg.Data, txData) { t.Fatal("estimating with wrong data") } return estimatedGasLimit, nil @@ -396,11 +396,11 @@ func TestTransactionSend(t *testing.T) { } return nil }), - backendmock.WithEstimateGasFunc(func(ctx context.Context, call ethereum.CallMsg) (gas uint64, err error) { - if !bytes.Equal(call.To.Bytes(), recipient.Bytes()) { - t.Fatalf("estimating with wrong recipient. wanted %x, got %x", recipient, call.To) + backendmock.WithEstimateGasAtBlockFunc(func(ctx context.Context, msg ethereum.CallMsg, blockNumber *big.Int) (gas uint64, err error) { + if !bytes.Equal(msg.To.Bytes(), recipient.Bytes()) { + t.Fatalf("estimating with wrong recipient. wanted %x, got %x", recipient, msg.To) } - if !bytes.Equal(call.Data, txData) { + if !bytes.Equal(msg.Data, txData) { t.Fatal("estimating with wrong data") } return estimatedGasLimit, nil @@ -461,7 +461,7 @@ func TestTransactionSend(t *testing.T) { } return nil }), - backendmock.WithEstimateGasFunc(func(ctx context.Context, call ethereum.CallMsg) (gas uint64, err error) { + backendmock.WithEstimateGasAtBlockFunc(func(ctx context.Context, call ethereum.CallMsg, blockNumber *big.Int) (gas uint64, err error) { if !bytes.Equal(call.To.Bytes(), recipient.Bytes()) { t.Fatalf("estimating with wrong recipient. wanted %x, got %x", recipient, call.To) } @@ -527,7 +527,7 @@ func TestTransactionSend(t *testing.T) { } return nil }), - backendmock.WithEstimateGasFunc(func(ctx context.Context, call ethereum.CallMsg) (gas uint64, err error) { + backendmock.WithEstimateGasAtBlockFunc(func(ctx context.Context, call ethereum.CallMsg, blockNumber *big.Int) (gas uint64, err error) { if !bytes.Equal(call.To.Bytes(), recipient.Bytes()) { t.Fatalf("estimating with wrong recipient. wanted %x, got %x", recipient, call.To) } diff --git a/pkg/transaction/wrapped/wrapped.go b/pkg/transaction/wrapped/wrapped.go index 1a51a618a88..1d4e452aef3 100644 --- a/pkg/transaction/wrapped/wrapped.go +++ b/pkg/transaction/wrapped/wrapped.go @@ -138,11 +138,10 @@ func (b *wrappedBackend) SuggestGasTipCap(ctx context.Context) (*big.Int, error) } return gasTipCap, nil } - -func (b *wrappedBackend) EstimateGas(ctx context.Context, call ethereum.CallMsg) (gas uint64, err error) { +func (b *wrappedBackend) EstimateGasAtBlock(ctx context.Context, msg ethereum.CallMsg, blockNumber *big.Int) (uint64, error) { b.metrics.TotalRPCCalls.Inc() b.metrics.EstimateGasCalls.Inc() - gas, err = b.backend.EstimateGas(ctx, call) + gas, err := b.backend.EstimateGasAtBlock(ctx, msg, blockNumber) if err != nil { b.metrics.TotalRPCErrors.Inc() return 0, err From ec5ff0d28f61cc8503acca1781b4d6b4371ae58f Mon Sep 17 00:00:00 2001 From: Calin M Date: Wed, 29 Oct 2025 09:30:31 +0200 Subject: [PATCH 37/96] Add validation for nat-addr (#5257) --- pkg/node/export_test.go | 7 +++ pkg/node/node.go | 40 +++++++++++++++++ pkg/node/node_test.go | 99 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 146 insertions(+) create mode 100644 pkg/node/export_test.go create mode 100644 pkg/node/node_test.go diff --git a/pkg/node/export_test.go b/pkg/node/export_test.go new file mode 100644 index 00000000000..3a142d94f7f --- /dev/null +++ b/pkg/node/export_test.go @@ -0,0 +1,7 @@ +// Copyright 2025 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package node + +var ValidatePublicAddress = validatePublicAddress diff --git a/pkg/node/node.go b/pkg/node/node.go index 163b599e651..02ca3f30521 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -20,6 +20,7 @@ import ( "net/http" "path/filepath" "runtime" + "strconv" "sync" "sync/atomic" "time" @@ -222,6 +223,10 @@ func NewBee( return nil, fmt.Errorf("tracer: %w", err) } + if err := validatePublicAddress(o.NATAddr); err != nil { + return nil, fmt.Errorf("invalid NAT address %s: %w", o.NATAddr, err) + } + ctx, ctxCancel := context.WithCancel(ctx) defer func() { // if there's been an error on this function @@ -1463,3 +1468,38 @@ func isChainEnabled(o *Options, swapEndpoint string, logger log.Logger) bool { logger.Info("starting with an enabled chain backend") return true // all other modes operate require chain enabled } + +func validatePublicAddress(addr string) error { + if addr == "" { + return nil + } + + host, port, err := net.SplitHostPort(addr) + if err != nil { + return fmt.Errorf("%w", err) + } + if host == "" { + return errors.New("host is empty") + } + if port == "" { + return errors.New("port is empty") + } + if _, err := strconv.ParseUint(port, 10, 16); err != nil { + return fmt.Errorf("port is not a valid number: %w", err) + } + if host == "localhost" { + return errors.New("localhost is not a valid address") + } + ip := net.ParseIP(host) + if ip == nil { + return errors.New("not a valid IP address") + } + if ip.IsLoopback() { + return errors.New("loopback address is not a valid address") + } + if ip.IsPrivate() { + return errors.New("private address is not a valid address") + } + + return nil +} diff --git a/pkg/node/node_test.go b/pkg/node/node_test.go new file mode 100644 index 00000000000..e051c2230b1 --- /dev/null +++ b/pkg/node/node_test.go @@ -0,0 +1,99 @@ +// Copyright 2025 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package node_test + +import ( + "testing" + + "github.com/ethersphere/bee/v2/pkg/node" +) + +func TestValidatePublicAddress(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + addr string + expErr bool + }{ + { + name: "empty host", + addr: ":1635", + expErr: true, + }, + { + name: "localhost", + addr: "localhost:1635", + expErr: true, + }, + { + name: "loopback", + addr: "127.0.0.1:1635", + expErr: true, + }, + { + name: "loopback ipv6", + addr: "[::1]:1635", + expErr: true, + }, + { + name: "missing port", + addr: "1.2.3.4", + expErr: true, + }, + { + name: "empty port", + addr: "1.2.3.4:", + expErr: true, + }, + { + name: "invalid port number", + addr: "1.2.3.4:abc", + expErr: true, + }, + { + name: "valid", + addr: "1.2.3.4:1635", + expErr: false, + }, + { + name: "valid ipv6", + addr: "[2001:db8::1]:1635", + expErr: false, + }, + { + name: "empty", + addr: "", + expErr: false, + }, + { + name: "invalid IP", + addr: "not-an-ip:8080", + expErr: true, + }, + { + name: "private IP", + addr: "192.168.1.1:8080", + expErr: true, + }, + { + name: "hostname", + addr: "example.com:8080", + expErr: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := node.ValidatePublicAddress(tc.addr) + if tc.expErr && err == nil { + t.Fatal("expected error, but got none") + } + if !tc.expErr && err != nil { + t.Fatalf("expected no error, but got: %v", err) + } + }) + } +} From dbf8c5e576f99e338dfde09bd08f85e0eb6e5785 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ljubi=C5=A1a=20Ga=C4=8Devi=C4=87?= <35105035+gacevicljubisa@users.noreply.github.com> Date: Mon, 3 Nov 2025 08:54:48 +0100 Subject: [PATCH 38/96] fix(transaction): improve legacy gasPrice to EIP-1559 dynamic fee (#5225) --- pkg/transaction/wrapped/fee.go | 34 ++++++++++++++++++++++++----- pkg/transaction/wrapped/fee_test.go | 7 ++++++ 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/pkg/transaction/wrapped/fee.go b/pkg/transaction/wrapped/fee.go index 3b02b3423ec..34fe2cc60d5 100644 --- a/pkg/transaction/wrapped/fee.go +++ b/pkg/transaction/wrapped/fee.go @@ -20,20 +20,44 @@ var ( ErrEIP1559NotSupported = errors.New("network does not appear to support EIP-1559 (no baseFee)") ) -// SuggestedFeeAndTip calculates the recommended gasFeeCap and gasTipCap for a transaction. +// SuggestedFeeAndTip calculates the recommended gasFeeCap (maxFeePerGas) and gasTipCap (maxPriorityFeePerGas) for a transaction. +// If gasPrice is provided (legacy mode): +// - On EIP-1559 networks: gasFeeCap = gasPrice; gasTipCap = max(gasPrice - baseFee, minimumTip) to respect the total cap while enforcing a tip floor where possible. +// - On pre-EIP-1559 networks: returns (gasPrice, gasPrice) for legacy transaction compatibility. +// +// If gasPrice is nil: Uses suggested tip with optional boost, enforces minimum, and sets gasFeeCap = 2 * baseFee + gasTipCap. func (b *wrappedBackend) SuggestedFeeAndTip(ctx context.Context, gasPrice *big.Int, boostPercent int) (*big.Int, *big.Int, error) { if gasPrice != nil { - // 1. gasFeeCap: The absolute maximum price per gas does not exceed the user's specified price. - // 2. gasTipCap: The entire amount (gasPrice - baseFee) can be used as a priority fee. - return new(big.Int).Set(gasPrice), new(big.Int).Set(gasPrice), nil + latestBlockHeader, err := b.backend.HeaderByNumber(ctx, nil) + if err != nil { + return nil, nil, fmt.Errorf("failed to get latest block header: %w", err) + } + if latestBlockHeader == nil || latestBlockHeader.BaseFee == nil { + return new(big.Int).Set(gasPrice), new(big.Int).Set(gasPrice), nil + } + + baseFee := latestBlockHeader.BaseFee + if gasPrice.Cmp(baseFee) < 0 { + return nil, nil, fmt.Errorf("specified gas price %s is below current base fee %s", gasPrice, baseFee) + } + + // nominal tip = gasPrice - baseFee + gasTipCap := new(big.Int).Sub(gasPrice, baseFee) + gasFeeCap := new(big.Int).Set(gasPrice) + + return gasFeeCap, gasTipCap, nil } gasTipCap, err := b.backend.SuggestGasTipCap(ctx) if err != nil { return nil, nil, fmt.Errorf("failed to suggest gas tip cap: %w", err) } + gasTipCap = new(big.Int).Set(gasTipCap) if boostPercent != 0 { + if boostPercent < 0 { + return nil, nil, fmt.Errorf("negative boostPercent (%d) not allowed", boostPercent) + } // multiplier: 100 + boostPercent (e.g., 110 for 10% boost) multiplier := new(big.Int).Add(big.NewInt(int64(percentageDivisor)), big.NewInt(int64(boostPercent))) // gasTipCap = gasTipCap * (100 + boostPercent) / 100 @@ -53,7 +77,7 @@ func (b *wrappedBackend) SuggestedFeeAndTip(ctx context.Context, gasPrice *big.I return nil, nil, ErrEIP1559NotSupported } - // EIP-1559: gasFeeCap = (2 * baseFee) + gasTipCap + // gasFeeCap = (2 * baseFee) + gasTipCap gasFeeCap := new(big.Int).Mul(latestBlockHeader.BaseFee, big.NewInt(int64(baseFeeMultiplier))) gasFeeCap.Add(gasFeeCap, gasTipCap) diff --git a/pkg/transaction/wrapped/fee_test.go b/pkg/transaction/wrapped/fee_test.go index 77847ec40d1..41afb69f28f 100644 --- a/pkg/transaction/wrapped/fee_test.go +++ b/pkg/transaction/wrapped/fee_test.go @@ -43,6 +43,13 @@ func TestSuggestedFeeAndTip(t *testing.T) { wantGasFeeCap: big.NewInt(1000), wantGasTipCap: big.NewInt(1000), }, + { + name: "with gas price and base fee", + gasPrice: big.NewInt(1000), + mockHeader: &types.Header{BaseFee: baseFee}, + wantGasFeeCap: big.NewInt(1000), + wantGasTipCap: big.NewInt(900), + }, { name: "suggest tip error", mockSuggestGasErr: errors.New("suggest tip error"), From 9fd1286660635148f56313f6be50604d21a0b915 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ljubi=C5=A1a=20Ga=C4=8Devi=C4=87?= <35105035+gacevicljubisa@users.noreply.github.com> Date: Mon, 3 Nov 2025 08:55:07 +0100 Subject: [PATCH 39/96] fix: add chain availability check (#5227) --- cmd/bee/cmd/db.go | 1 - pkg/api/router.go | 18 +++++ pkg/node/chain.go | 108 +++++-------------------- pkg/node/node.go | 36 ++++----- pkg/transaction/backendnoop/backend.go | 94 +++++++++++++++++++++ pkg/transaction/transaction.go | 3 +- 6 files changed, 148 insertions(+), 112 deletions(-) create mode 100644 pkg/transaction/backendnoop/backend.go diff --git a/cmd/bee/cmd/db.go b/cmd/bee/cmd/db.go index 62eee2591c2..8bca2ed31cc 100644 --- a/cmd/bee/cmd/db.go +++ b/cmd/bee/cmd/db.go @@ -31,7 +31,6 @@ import ( const ( optionNameValidation = "validate" - optionNameValidationPin = "validate-pin" optionNameCollectionPin = "pin" optionNameOutputLocation = "output" ) diff --git a/pkg/api/router.go b/pkg/api/router.go index 13a782bcd77..776def9457d 100644 --- a/pkg/api/router.go +++ b/pkg/api/router.go @@ -14,6 +14,7 @@ import ( "github.com/ethersphere/bee/v2/pkg/jsonhttp" "github.com/ethersphere/bee/v2/pkg/log/httpaccess" "github.com/ethersphere/bee/v2/pkg/swarm" + "github.com/ethersphere/bee/v2/pkg/transaction/backendnoop" "github.com/felixge/fgprof" "github.com/gorilla/handlers" "github.com/gorilla/mux" @@ -218,6 +219,17 @@ func (s *Service) checkStorageIncentivesAvailability(handler http.Handler) http. }) } +func (s *Service) checkChainAvailability(handler http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if _, ok := s.chainBackend.(*backendnoop.Backend); ok { + jsonhttp.Forbidden(w, "Chain is disabled. This endpoint is unavailable.") + return + } + + handler.ServeHTTP(w, r) + }) +} + func (s *Service) mountAPI() { subdomainRouter := s.router.Host("{subdomain:.*}.swarm.localhost").Subrouter() @@ -564,6 +576,7 @@ func (s *Service) mountBusinessDebug() { )) handle("/stamps", web.ChainHandlers( + s.checkChainAvailability, s.postageSyncStatusCheckHandler, web.FinalHandler(jsonhttp.MethodHandler{ "GET": http.HandlerFunc(s.postageGetStampsHandler), @@ -571,6 +584,7 @@ func (s *Service) mountBusinessDebug() { ) handle("/stamps/{batch_id}", web.ChainHandlers( + s.checkChainAvailability, s.postageSyncStatusCheckHandler, web.FinalHandler(jsonhttp.MethodHandler{ "GET": http.HandlerFunc(s.postageGetStampHandler), @@ -578,6 +592,7 @@ func (s *Service) mountBusinessDebug() { ) handle("/stamps/{batch_id}/buckets", web.ChainHandlers( + s.checkChainAvailability, s.postageSyncStatusCheckHandler, web.FinalHandler(jsonhttp.MethodHandler{ "GET": http.HandlerFunc(s.postageGetStampBucketsHandler), @@ -585,6 +600,7 @@ func (s *Service) mountBusinessDebug() { ) handle("/stamps/{amount}/{depth}", web.ChainHandlers( + s.checkChainAvailability, s.postageAccessHandler, s.postageSyncStatusCheckHandler, s.gasConfigMiddleware("create batch"), @@ -594,6 +610,7 @@ func (s *Service) mountBusinessDebug() { ) handle("/stamps/topup/{batch_id}/{amount}", web.ChainHandlers( + s.checkChainAvailability, s.postageAccessHandler, s.postageSyncStatusCheckHandler, s.gasConfigMiddleware("topup batch"), @@ -603,6 +620,7 @@ func (s *Service) mountBusinessDebug() { ) handle("/stamps/dilute/{batch_id}/{depth}", web.ChainHandlers( + s.checkChainAvailability, s.postageAccessHandler, s.postageSyncStatusCheckHandler, s.gasConfigMiddleware("dilute batch"), diff --git a/pkg/node/chain.go b/pkg/node/chain.go index e6fc2c429ec..78fc0a5448d 100644 --- a/pkg/node/chain.go +++ b/pkg/node/chain.go @@ -13,9 +13,7 @@ import ( "strings" "time" - "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/rpc" "github.com/ethersphere/bee/v2/pkg/config" @@ -31,8 +29,8 @@ import ( "github.com/ethersphere/bee/v2/pkg/settlement/swap/swapprotocol" "github.com/ethersphere/bee/v2/pkg/storage" "github.com/ethersphere/bee/v2/pkg/transaction" + "github.com/ethersphere/bee/v2/pkg/transaction/backendnoop" "github.com/ethersphere/bee/v2/pkg/transaction/wrapped" - "github.com/prometheus/client_golang/prometheus" ) const ( @@ -48,28 +46,28 @@ func InitChain( logger log.Logger, stateStore storage.StateStorer, endpoint string, - oChainID int64, + chainID int64, signer crypto.Signer, pollingInterval time.Duration, chainEnabled bool, minimumGasTipCap uint64, ) (transaction.Backend, common.Address, int64, transaction.Monitor, transaction.Service, error) { - var backend transaction.Backend = &noOpChainBackend{ - chainID: oChainID, - } + backend := backendnoop.New(chainID) if chainEnabled { - // connect to the real one rpcClient, err := rpc.DialContext(ctx, endpoint) if err != nil { return nil, common.Address{}, 0, nil, nil, fmt.Errorf("dial blockchain client: %w", err) } var versionString string - err = rpcClient.CallContext(ctx, &versionString, "web3_clientVersion") - if err != nil { - logger.Info("could not connect to backend; in a swap-enabled network a working blockchain node (for xdai network in production, sepolia in testnet) is required; check your node or specify another node using --blockchain-rpc-endpoint.", "backend_endpoint", endpoint) - return nil, common.Address{}, 0, nil, nil, fmt.Errorf("blockchain client get version: %w", err) + if err = rpcClient.CallContext(ctx, &versionString, "web3_clientVersion"); err != nil { + logger.Info("could not connect to backend; "+ + "in a swap-enabled network a working blockchain node "+ + "(for xDAI network in production, SepoliaETH in testnet) is required; "+ + "check your node or specify another node using --blockchain-rpc-endpoint.", + "blockchain-rpc-endpoint", endpoint) + return nil, common.Address{}, 0, nil, nil, fmt.Errorf("get client version: %w", err) } logger.Info("connected to blockchain backend", "version", versionString) @@ -77,9 +75,13 @@ func InitChain( backend = wrapped.NewBackend(ethclient.NewClient(rpcClient), minimumGasTipCap) } - chainID, err := backend.ChainID(ctx) + backendChainID, err := backend.ChainID(ctx) if err != nil { - return nil, common.Address{}, 0, nil, nil, fmt.Errorf("get chain id: %w", err) + return nil, common.Address{}, 0, nil, nil, fmt.Errorf("getting chain id: %w", err) + } + + if chainID != -1 && chainID != backendChainID.Int64() { + return nil, common.Address{}, 0, nil, nil, fmt.Errorf("connected to wrong network: expected chain id %d, got %d", chainID, backendChainID.Int64()) } overlayEthAddress, err := signer.EthereumAddress() @@ -89,18 +91,19 @@ func InitChain( transactionMonitor := transaction.NewMonitor(logger, backend, overlayEthAddress, pollingInterval, cancellationDepth) - transactionService, err := transaction.NewService(logger, overlayEthAddress, backend, signer, stateStore, chainID, transactionMonitor) + transactionService, err := transaction.NewService(logger, overlayEthAddress, backend, signer, stateStore, backendChainID, transactionMonitor) if err != nil { - return nil, common.Address{}, 0, nil, nil, fmt.Errorf("new transaction service: %w", err) + return nil, common.Address{}, 0, nil, nil, fmt.Errorf("transaction service: %w", err) } - return backend, overlayEthAddress, chainID.Int64(), transactionMonitor, transactionService, nil + return backend, overlayEthAddress, backendChainID.Int64(), transactionMonitor, transactionService, nil } // InitChequebookFactory will initialize the chequebook factory with the given // chain backend. func InitChequebookFactory(logger log.Logger, backend transaction.Backend, chainID int64, transactionService transaction.Service, factoryAddress string) (chequebook.Factory, error) { var currentFactory common.Address + chainCfg, found := config.GetByChainID(chainID) foundFactory := chainCfg.CurrentFactoryAddress @@ -340,74 +343,3 @@ func (m *noOpChequebookService) LastCheque(common.Address) (*chequebook.SignedCh func (m *noOpChequebookService) LastCheques() (map[common.Address]*chequebook.SignedCheque, error) { return nil, postagecontract.ErrChainDisabled } - -// noOpChainBackend is a noOp implementation for transaction.Backend interface. -type noOpChainBackend struct { - chainID int64 -} - -func (m noOpChainBackend) Metrics() []prometheus.Collector { - return nil -} - -func (m noOpChainBackend) CallContract(context.Context, ethereum.CallMsg, *big.Int) ([]byte, error) { - return nil, errors.New("disabled chain backend") -} - -func (m noOpChainBackend) HeaderByNumber(context.Context, *big.Int) (*types.Header, error) { - h := new(types.Header) - h.Time = uint64(time.Now().Unix()) - return h, nil -} - -func (m noOpChainBackend) PendingNonceAt(context.Context, common.Address) (uint64, error) { - panic("chain no op: PendingNonceAt") -} - -func (m noOpChainBackend) SuggestedFeeAndTip(ctx context.Context, gasPrice *big.Int, boostPercent int) (*big.Int, *big.Int, error) { - panic("chain no op: SuggestedFeeAndTip") -} - -func (m noOpChainBackend) SuggestGasTipCap(context.Context) (*big.Int, error) { - panic("chain no op: SuggestGasTipCap") -} - -func (m noOpChainBackend) EstimateGasAtBlock(context.Context, ethereum.CallMsg, *big.Int) (uint64, error) { - panic("chain no op: EstimateGasAtBlock") -} - -func (m noOpChainBackend) SendTransaction(context.Context, *types.Transaction) error { - panic("chain no op: SendTransaction") -} - -func (m noOpChainBackend) TransactionReceipt(context.Context, common.Hash) (*types.Receipt, error) { - r := new(types.Receipt) - r.BlockNumber = big.NewInt(1) - return r, nil -} - -func (m noOpChainBackend) TransactionByHash(context.Context, common.Hash) (tx *types.Transaction, isPending bool, err error) { - panic("chain no op: TransactionByHash") -} - -func (m noOpChainBackend) BlockNumber(context.Context) (uint64, error) { - return 4, nil -} - -func (m noOpChainBackend) BalanceAt(context.Context, common.Address, *big.Int) (*big.Int, error) { - return nil, postagecontract.ErrChainDisabled -} - -func (m noOpChainBackend) NonceAt(context.Context, common.Address, *big.Int) (uint64, error) { - panic("chain no op: NonceAt") -} - -func (m noOpChainBackend) FilterLogs(context.Context, ethereum.FilterQuery) ([]types.Log, error) { - panic("chain no op: FilterLogs") -} - -func (m noOpChainBackend) ChainID(context.Context) (*big.Int, error) { - return big.NewInt(m.chainID), nil -} - -func (m noOpChainBackend) Close() {} diff --git a/pkg/node/node.go b/pkg/node/node.go index 02ca3f30521..d5ba02bdc56 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -363,16 +363,10 @@ func NewBee( } var ( - chainBackend transaction.Backend - overlayEthAddress common.Address - chainID int64 - transactionService transaction.Service - transactionMonitor transaction.Monitor - chequebookFactory chequebook.Factory - chequebookService chequebook.Service = new(noOpChequebookService) - chequeStore chequebook.ChequeStore - cashoutService chequebook.CashoutService - erc20Service erc20.Service + chequebookService chequebook.Service = new(noOpChequebookService) + chequeStore chequebook.ChequeStore + cashoutService chequebook.CashoutService + erc20Service erc20.Service ) chainEnabled := isChainEnabled(o, o.BlockchainRpcEndpoint, logger) @@ -394,7 +388,7 @@ func NewBee( } } - chainBackend, overlayEthAddress, chainID, transactionMonitor, transactionService, err = InitChain( + chainBackend, overlayEthAddress, chainID, transactionMonitor, transactionService, err := InitChain( ctx, logger, stateStore, @@ -408,14 +402,10 @@ func NewBee( if err != nil { return nil, fmt.Errorf("init chain: %w", err) } - b.ethClientCloser = chainBackend.Close - - logger.Info("using chain with network network", "chain_id", chainID, "network_id", networkID) - if o.ChainID != -1 && o.ChainID != chainID { - return nil, fmt.Errorf("connected to wrong blockchain network; network chainID %d; configured chainID %d", chainID, o.ChainID) - } + logger.Info("using chain with network", "chain_id", chainID, "network_id", networkID) + b.ethClientCloser = chainBackend.Close b.transactionCloser = tracerCloser b.transactionMonitorCloser = transactionMonitor @@ -514,7 +504,7 @@ func NewBee( } if o.SwapEnable { - chequebookFactory, err = InitChequebookFactory(logger, chainBackend, chainID, transactionService, o.SwapFactoryAddress) + chequebookFactory, err := InitChequebookFactory(logger, chainBackend, chainID, transactionService, o.SwapFactoryAddress) if err != nil { return nil, fmt.Errorf("init chequebook factory: %w", err) } @@ -1460,12 +1450,16 @@ func isChainEnabled(o *Options, swapEndpoint string, logger log.Logger) bool { chainDisabled := swapEndpoint == "" lightMode := !o.FullNodeMode - if lightMode && chainDisabled { // ultra light mode is LightNode mode with chain disabled - logger.Info("starting with a disabled chain backend") + if lightMode && chainDisabled { + logger.Info("chain backend disabled - starting in ultra-light mode", + "full_node_mode", o.FullNodeMode, + "blockchain-rpc-endpoint", swapEndpoint) return false } - logger.Info("starting with an enabled chain backend") + logger.Info("chain backend enabled - blockchain functionality available", + "full_node_mode", o.FullNodeMode, + "blockchain-rpc-endpoint", swapEndpoint) return true // all other modes operate require chain enabled } diff --git a/pkg/transaction/backendnoop/backend.go b/pkg/transaction/backendnoop/backend.go new file mode 100644 index 00000000000..cc09b459bd2 --- /dev/null +++ b/pkg/transaction/backendnoop/backend.go @@ -0,0 +1,94 @@ +// Copyright 2025 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package backendnoop + +import ( + "context" + "math/big" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethersphere/bee/v2/pkg/postage/postagecontract" + "github.com/ethersphere/bee/v2/pkg/transaction" + "github.com/prometheus/client_golang/prometheus" +) + +var _ transaction.Backend = (*Backend)(nil) + +// Backend is a no-op implementation for transaction.Backend interface. +// It's used when the blockchain functionality is disabled. +type Backend struct { + chainID int64 +} + +// New creates a new no-op backend with the specified chain ID. +func New(chainID int64) transaction.Backend { + return &Backend{ + chainID: chainID, + } +} + +func (b *Backend) Metrics() []prometheus.Collector { + return nil +} + +func (b *Backend) CallContract(context.Context, ethereum.CallMsg, *big.Int) ([]byte, error) { + return nil, postagecontract.ErrChainDisabled +} + +func (b *Backend) HeaderByNumber(context.Context, *big.Int) (*types.Header, error) { + return nil, postagecontract.ErrChainDisabled +} + +func (b *Backend) PendingNonceAt(context.Context, common.Address) (uint64, error) { + return 0, postagecontract.ErrChainDisabled +} + +func (b *Backend) SuggestedFeeAndTip(ctx context.Context, gasPrice *big.Int, boostPercent int) (*big.Int, *big.Int, error) { + return nil, nil, postagecontract.ErrChainDisabled +} + +func (b *Backend) SuggestGasTipCap(context.Context) (*big.Int, error) { + return nil, postagecontract.ErrChainDisabled +} + +func (b *Backend) EstimateGasAtBlock(ctx context.Context, msg ethereum.CallMsg, blockNumber *big.Int) (uint64, error) { + return 0, postagecontract.ErrChainDisabled +} + +func (b *Backend) SendTransaction(context.Context, *types.Transaction) error { + return postagecontract.ErrChainDisabled +} + +func (b *Backend) TransactionReceipt(context.Context, common.Hash) (*types.Receipt, error) { + return nil, postagecontract.ErrChainDisabled +} + +func (b *Backend) TransactionByHash(context.Context, common.Hash) (tx *types.Transaction, isPending bool, err error) { + return nil, false, postagecontract.ErrChainDisabled +} + +func (b *Backend) BlockNumber(context.Context) (uint64, error) { + return 0, postagecontract.ErrChainDisabled +} + +func (b *Backend) BalanceAt(context.Context, common.Address, *big.Int) (*big.Int, error) { + return nil, postagecontract.ErrChainDisabled +} + +func (b *Backend) NonceAt(context.Context, common.Address, *big.Int) (uint64, error) { + return 0, postagecontract.ErrChainDisabled +} + +func (b *Backend) FilterLogs(context.Context, ethereum.FilterQuery) ([]types.Log, error) { + return nil, postagecontract.ErrChainDisabled +} + +func (b *Backend) ChainID(context.Context) (*big.Int, error) { + return big.NewInt(b.chainID), nil +} + +func (b *Backend) Close() {} diff --git a/pkg/transaction/transaction.go b/pkg/transaction/transaction.go index 307d1c46261..cc88e7a3ab4 100644 --- a/pkg/transaction/transaction.go +++ b/pkg/transaction/transaction.go @@ -141,8 +141,7 @@ func NewService(logger log.Logger, overlayEthAddress common.Address, backend Bac monitor: monitor, } - err = t.waitForAllPendingTx() - if err != nil { + if err = t.waitForAllPendingTx(); err != nil { return nil, err } From 4ab44aa4ec2ade57ed1ff49e02fd27d17a624e5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ljubi=C5=A1a=20Ga=C4=8Devi=C4=87?= <35105035+gacevicljubisa@users.noreply.github.com> Date: Mon, 3 Nov 2025 10:33:32 +0100 Subject: [PATCH 40/96] fix(pushsync): cap overdraft backoff delay with jitter (#5270) --- pkg/pushsync/backoff_test.go | 82 ++++++++++++++++++++++++++++++++++++ pkg/pushsync/export_test.go | 11 +++-- pkg/pushsync/pushsync.go | 58 ++++++++++++++++++++----- 3 files changed, 137 insertions(+), 14 deletions(-) create mode 100644 pkg/pushsync/backoff_test.go diff --git a/pkg/pushsync/backoff_test.go b/pkg/pushsync/backoff_test.go new file mode 100644 index 00000000000..67b1fa668d4 --- /dev/null +++ b/pkg/pushsync/backoff_test.go @@ -0,0 +1,82 @@ +// Copyright 2025 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pushsync_test + +import ( + "testing" + "time" + + "github.com/ethersphere/bee/v2/pkg/pushsync" +) + +func TestCalculateOverdraftBackoff_InitialBounds(t *testing.T) { + const samples = 200 + + base := time.Duration(float64(pushsync.OverDraftRefresh) * pushsync.OverDraftBackoffMultiplier) + lower := time.Duration(float64(base) * (1 - pushsync.OverDraftJitterPercent)) + upper := time.Duration(float64(base) * (1 + pushsync.OverDraftJitterPercent)) + + for range samples { + got := pushsync.CalculateOverdraftBackoff(0) + if got < lower { + t.Fatalf("backoff below lower bound: got=%v lower=%v", got, lower) + } + if got > upper { + t.Fatalf("backoff above upper bound: got=%v upper=%v", got, upper) + } + if got < pushsync.OverDraftRefresh { + t.Fatalf("backoff below minimum refresh: got=%v min=%v", got, pushsync.OverDraftRefresh) + } + } +} + +func TestCalculateOverdraftBackoff_MaxEdgeJittersAroundCap(t *testing.T) { + const samples = 400 + + current := pushsync.MaxOverDraftRefresh * 10 + base := pushsync.MaxOverDraftRefresh + lower := time.Duration(float64(base) * (1 - pushsync.OverDraftJitterPercent)) + upper := time.Duration(float64(base) * (1 + pushsync.OverDraftJitterPercent)) + + sawAboveMax := false + sawBelowMax := false + + for range samples { + got := pushsync.CalculateOverdraftBackoff(current) + if got < lower { + t.Fatalf("backoff below expected jitter band: got=%v lower=%v", got, lower) + } + if got > upper { + t.Fatalf("backoff above expected jitter band: got=%v upper=%v", got, upper) + } + if got > base { + sawAboveMax = true + } + if got < base { + sawBelowMax = true + } + } + + if !sawAboveMax { + t.Fatalf("expected at least one sample above maxOverDraftRefresh; base=%v", base) + } + if !sawBelowMax { + t.Fatalf("expected at least one sample below maxOverDraftRefresh; base=%v", base) + } + t.Logf("seen above max: %v, seen below max: %v", sawAboveMax, sawBelowMax) +} + +func TestCalculateOverdraftBackoff_NeverBelowMin(t *testing.T) { + const samples = 200 + for i := range samples { + got := pushsync.CalculateOverdraftBackoff(0) + if got < pushsync.OverDraftRefresh { + t.Fatalf("backoff below minimum refresh: got=%v min=%v", got, pushsync.OverDraftRefresh) + } + if i < 5 { + t.Logf("min-check sample %d: delay=%v (min %v)", i, got, pushsync.OverDraftRefresh) + } + } +} diff --git a/pkg/pushsync/export_test.go b/pkg/pushsync/export_test.go index 840ce5b7849..778b76ea97f 100644 --- a/pkg/pushsync/export_test.go +++ b/pkg/pushsync/export_test.go @@ -5,7 +5,12 @@ package pushsync var ( - ProtocolName = protocolName - ProtocolVersion = protocolVersion - StreamName = streamName + ProtocolName = protocolName + ProtocolVersion = protocolVersion + StreamName = streamName + OverDraftRefresh = overDraftRefresh + MaxOverDraftRefresh = maxOverDraftRefresh + OverDraftBackoffMultiplier = overDraftBackoffMultiplier + OverDraftJitterPercent = overDraftJitterPercent + CalculateOverdraftBackoff = calculateOverdraftBackoff ) diff --git a/pkg/pushsync/pushsync.go b/pkg/pushsync/pushsync.go index 9a9754a9ab3..ba10a5c4403 100644 --- a/pkg/pushsync/pushsync.go +++ b/pkg/pushsync/pushsync.go @@ -10,6 +10,7 @@ import ( "context" "errors" "fmt" + "math/rand/v2" "strconv" "time" @@ -44,10 +45,13 @@ const ( ) const ( - defaultTTL = 30 * time.Second // request time to live - preemptiveInterval = 5 * time.Second // P90 request time to live - skiplistDur = 5 * time.Minute - overDraftRefresh = time.Millisecond * 600 + defaultTTL = 30 * time.Second // request time to live + preemptiveInterval = 5 * time.Second // P90 request time to live + skiplistDur = 5 * time.Minute + overDraftRefresh = time.Millisecond * 600 // overdraft refresh duration and initial backoff delay + maxOverDraftRefresh = time.Second * 5 // maximum backoff delay + overDraftBackoffMultiplier = 1.2 // backoff multiplier + overDraftJitterPercent = 0.1 // 10% jitter ) const ( @@ -315,7 +319,6 @@ func (ps *PushSync) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream) default: ps.metrics.Forwarder.Inc() return fmt.Errorf("handler: push to closest chunk %s: %w", chunkAddress, err) - } } @@ -355,10 +358,12 @@ func (ps *PushSync) pushToClosest(ctx context.Context, ch swarm.Chunk, origin bo ps.metrics.TotalRequests.Inc() var ( - sentErrorsLeft = 1 - preemptiveTicker <-chan time.Time - inflight int - parallelForwards = maxMultiplexForwards + sentErrorsLeft = 1 + preemptiveTicker <-chan time.Time + inflight int + parallelForwards = maxMultiplexForwards + overdraftBackoffDelay time.Duration + overdraftRetryCount int ) if origin { @@ -424,10 +429,19 @@ func (ps *PushSync) pushToClosest(ctx context.Context, ch swarm.Chunk, origin bo continue // there is still an inflight request, wait for it's result } - ps.logger.Debug("sleeping to refresh overdraft balance", "chunk_address", ch.Address()) + overdraftBackoffDelay = calculateOverdraftBackoff(overdraftBackoffDelay) + overdraftRetryCount++ + + // log first 3 attempts and then every 5th + if overdraftRetryCount%5 == 1 || overdraftRetryCount <= 3 { + ps.logger.Debug("sleeping to refresh overdraft balance", + "chunk_address", ch.Address(), + "backoff_delay", overdraftBackoffDelay, + "retry_count", overdraftRetryCount) + } select { - case <-time.After(overDraftRefresh): + case <-time.After(overdraftBackoffDelay): retry() continue case <-ctx.Done(): @@ -444,6 +458,12 @@ func (ps *PushSync) pushToClosest(ctx context.Context, ch swarm.Chunk, origin bo continue } + if overdraftRetryCount > 0 { + ps.logger.Debug("overdraft resolved, resetting backoff", "chunk_address", ch.Address(), "total_retries", overdraftRetryCount) + overdraftBackoffDelay = 0 + overdraftRetryCount = 0 + } + // since we can reach into the neighborhood of the chunk // act as the multiplexer and push the chunk in parallel to multiple peers if swarm.Proximity(peer.Bytes(), ch.Address().Bytes()) >= rad { @@ -678,6 +698,22 @@ func (ps *PushSync) validStampWrapper(f postage.ValidStampFn) postage.ValidStamp } } +// calculateOverdraftBackoff calculates the next backoff delay using exponential backoff with jitter. +// It takes the current backoff delay and returns the next delay, capped at maxOverDraftRefresh. +func calculateOverdraftBackoff(currentDelay time.Duration) time.Duration { + if currentDelay == 0 { + currentDelay = overDraftRefresh + } + + nextDelay := min(time.Duration(float64(currentDelay)*overDraftBackoffMultiplier), maxOverDraftRefresh) + + jitterRange := float64(nextDelay) * overDraftJitterPercent + jitter := (rand.Float64() - 0.5) * 2 * jitterRange + finalDelay := max(time.Duration(float64(nextDelay)+jitter), overDraftRefresh) + + return finalDelay +} + func (s *PushSync) Close() error { return s.errSkip.Close() } From 4e2afdf10354bb1239a2140deaaa258a232725b3 Mon Sep 17 00:00:00 2001 From: Akrem Chabchoub <121046693+akrem-chabchoub@users.noreply.github.com> Date: Mon, 3 Nov 2025 11:15:38 +0100 Subject: [PATCH 41/96] docs(docker-readme): format and fix code blocks to prevent warnings (#5273) --- packaging/docker/README.md | 21 ++++++++++++++++----- pkg/storageincentives/agent.go | 2 +- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/packaging/docker/README.md b/packaging/docker/README.md index bce886018ae..db118ff1ef0 100644 --- a/packaging/docker/README.md +++ b/packaging/docker/README.md @@ -2,49 +2,60 @@ The docker-compose provides an app container for Bee. To prepare your machine to run docker compose execute -``` + +```bash mkdir -p bee && cd bee wget -q https://raw.githubusercontent.com/ethersphere/bee/master/packaging/docker/docker-compose.yml wget -q https://raw.githubusercontent.com/ethersphere/bee/master/packaging/docker/env -O .env ``` + Set all configuration variables inside `.env` If you want to run node in full mode, set `BEE_FULL_NODE=true` Bee requires an Ethereum endpoint to function. Obtain a free Infura account and set: + - `BEE_BLOCKCHAIN_RPC_ENDPOINT=wss://sepolia.infura.io/ws/v3/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx` Set bee password by either setting `BEE_PASSWORD` or `BEE_PASSWORD_FILE` If you want to use password file set it to + - `BEE_PASSWORD_FILE=/password` Mount password file local file system by adding -``` + +```yaml - ./password:/password ``` + to bee volumes inside `docker-compose.yml` Start it with -``` + +```bash docker-compose up -d ``` From logs find URL line with `on sepolia you can get both sepolia eth and sepolia bzz from` and prefund your node -``` + +```bash docker-compose logs -f bee-1 ``` Update services with -``` + +```bash docker-compose pull && docker-compose up -d ``` ## Running multiple Bee nodes + It is easy to run multiple bee nodes with docker compose by adding more services to `docker-compose.yaml` To do so, open `docker-compose.yaml`, copy lines 4-54 and past this after line 54 (whole bee-1 section). In the copied lines, replace all occurrences of `bee-1` with `bee-2` and adjust the `API_ADDR` and `P2P_ADDR` to respectively `1733`, `1734.` Lastly, add your newly configured services under `volumes` (last lines), such that it looks like: + ```yaml volumes: bee-1: diff --git a/pkg/storageincentives/agent.go b/pkg/storageincentives/agent.go index 04c906c132a..0b3feb31ea6 100644 --- a/pkg/storageincentives/agent.go +++ b/pkg/storageincentives/agent.go @@ -417,7 +417,7 @@ func (a *Agent) handleSample(ctx context.Context, round uint64) (bool, error) { } if !a.state.IsHealthy() { - a.logger.Info("skipping round because node is unhealhy", "round", round) + a.logger.Info("skipping round because node is unhealthy", "round", round) return false, nil } From 4deaa1ef7e28054278b887697835d79e139ae87b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Nov 2025 12:45:35 +0100 Subject: [PATCH 42/96] chore(deps): bump github.com/consensys/gnark-crypto from 0.16.0 to 0.18.1 (#5271) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 7 ++----- go.sum | 15 ++++----------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index 5e74fa8aed6..a9b5ca834b2 100644 --- a/go.mod +++ b/go.mod @@ -67,8 +67,7 @@ require ( github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/codahale/hdrhistogram v0.0.0-00010101000000-000000000000 // indirect - github.com/consensys/bavard v0.1.27 // indirect - github.com/consensys/gnark-crypto v0.16.0 // indirect + github.com/consensys/gnark-crypto v0.18.1 // indirect github.com/containerd/cgroups v1.1.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/crate-crypto/go-eth-kzg v1.3.0 // indirect @@ -125,7 +124,6 @@ require ( github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/mapstructure v1.4.1 // indirect - github.com/mmcloughlin/addchain v0.4.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect @@ -153,7 +151,7 @@ require ( github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/cast v1.3.0 // indirect github.com/spf13/jwalterweatherman v1.0.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/pflag v1.0.6 // indirect github.com/subosito/gotenv v1.2.0 // indirect github.com/supranational/blst v0.3.14 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect @@ -176,7 +174,6 @@ require ( gopkg.in/ini.v1 v1.57.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.2.1 // indirect - rsc.io/tmplfunc v0.0.3 // indirect ) replace github.com/codahale/hdrhistogram => github.com/HdrHistogram/hdrhistogram-go v0.0.0-20200919145931-8dac23c8dac1 diff --git a/go.sum b/go.sum index b07fdc93fc9..4badb0a9ca9 100644 --- a/go.sum +++ b/go.sum @@ -175,11 +175,9 @@ github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZ github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= -github.com/consensys/bavard v0.1.27 h1:j6hKUrGAy/H+gpNrpLU3I26n1yc+VMGmd6ID5+gAhOs= -github.com/consensys/bavard v0.1.27/go.mod h1:k/zVjHHC4B+PQy1Pg7fgvG3ALicQw540Crag8qx+dZs= github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1:815PAHg3wvysy0SyIqanF8gZ0Y1wjk/hrDHD/iT88+Q= -github.com/consensys/gnark-crypto v0.16.0 h1:8Dl4eYmUWK9WmlP1Bj6je688gBRJCJbT8Mw4KoTAawo= -github.com/consensys/gnark-crypto v0.16.0/go.mod h1:Ke3j06ndtPTVvo++PhGNgvm+lgpLvzbcE2MqljY7diU= +github.com/consensys/gnark-crypto v0.18.1 h1:RyLV6UhPRoYYzaFnPQA4qK3DyuDgkTgskDdoGqFt3fI= +github.com/consensys/gnark-crypto v0.18.1/go.mod h1:L3mXGFTe1ZN+RSJ+CLjUt9x7PNdx8ubaYfDROyp2Z8c= github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= @@ -413,7 +411,6 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 h1:y3N7Bm7Y9/CtpiVkw/ZWj6lSlDF3F74SfKwfTCer72Q= github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.5/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -663,9 +660,6 @@ github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxd github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/pointerstructure v1.2.0 h1:O+i9nHnXS3l/9Wu7r4NrEdwA2VFTicjUEN1uBnDo34A= github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4= -github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= -github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU= -github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -913,8 +907,9 @@ github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3k github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= @@ -1500,7 +1495,5 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU= -rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA= sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= From 53d66ba6118c8d4fb0281fd8fb46f5ef4136cb38 Mon Sep 17 00:00:00 2001 From: sbackend123 Date: Mon, 3 Nov 2025 14:20:51 +0100 Subject: [PATCH 43/96] feat: multiple underlay addresses (#5204) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Janoš Guljaš Co-authored-by: Ljubisa Gacevic Co-authored-by: Calin Martinconi --- go.mod | 2 +- pkg/addressbook/addressbook_test.go | 2 +- pkg/api/peer.go | 3 +- pkg/api/peer_test.go | 18 +- pkg/bzz/address.go | 139 +++++++-- pkg/bzz/address_test.go | 244 +++++++++++++++- pkg/bzz/export_test.go | 7 + pkg/bzz/underlay.go | 98 +++++++ pkg/bzz/underlay_test.go | 223 +++++++++++++++ pkg/bzz/utilities_test.go | 33 ++- pkg/hive/hive.go | 65 ++++- pkg/hive/hive_test.go | 132 +++++++-- pkg/node/devnode.go | 2 +- pkg/node/node.go | 2 +- pkg/p2p/libp2p/connections_test.go | 33 +-- .../libp2p/internal/handshake/handshake.go | 171 ++++++++---- .../internal/handshake/handshake_test.go | 85 +++--- pkg/p2p/libp2p/internal/reacher/metrics.go | 38 --- pkg/p2p/libp2p/internal/reacher/reacher.go | 41 ++- .../libp2p/internal/reacher/reacher_test.go | 11 +- pkg/p2p/libp2p/libp2p.go | 263 +++++++++++++++--- pkg/p2p/libp2p/libp2p_test.go | 4 +- pkg/p2p/libp2p/protocols_test.go | 2 +- pkg/p2p/mock/mock.go | 6 +- pkg/p2p/p2p.go | 2 +- pkg/topology/kademlia/kademlia.go | 22 +- pkg/topology/kademlia/kademlia_test.go | 60 ++-- 27 files changed, 1387 insertions(+), 321 deletions(-) create mode 100644 pkg/bzz/export_test.go create mode 100644 pkg/bzz/underlay.go create mode 100644 pkg/bzz/underlay_test.go delete mode 100644 pkg/p2p/libp2p/internal/reacher/metrics.go diff --git a/go.mod b/go.mod index a9b5ca834b2..adae83026a2 100644 --- a/go.mod +++ b/go.mod @@ -32,6 +32,7 @@ require ( github.com/multiformats/go-multiaddr-dns v0.3.1 github.com/multiformats/go-multihash v0.2.3 github.com/multiformats/go-multistream v0.5.0 + github.com/multiformats/go-varint v0.0.7 github.com/opentracing/opentracing-go v1.2.0 github.com/prometheus/client_golang v1.21.1 github.com/spf13/afero v1.6.0 @@ -130,7 +131,6 @@ require ( github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect github.com/multiformats/go-multibase v0.2.0 // indirect github.com/multiformats/go-multicodec v0.9.0 // indirect - github.com/multiformats/go-varint v0.0.7 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/onsi/ginkgo/v2 v2.15.0 // indirect github.com/opencontainers/runtime-spec v1.2.0 // indirect diff --git a/pkg/addressbook/addressbook_test.go b/pkg/addressbook/addressbook_test.go index 1b5c22490b5..1307ad92b67 100644 --- a/pkg/addressbook/addressbook_test.go +++ b/pkg/addressbook/addressbook_test.go @@ -47,7 +47,7 @@ func run(t *testing.T, f bookFunc) { t.Fatal(err) } - bzzAddr, err := bzz.NewAddress(crypto.NewDefaultSigner(pk), multiaddr, addr1, 1, trxHash) + bzzAddr, err := bzz.NewAddress(crypto.NewDefaultSigner(pk), []ma.Multiaddr{multiaddr}, addr1, 1, trxHash) if err != nil { t.Fatal(err) } diff --git a/pkg/api/peer.go b/pkg/api/peer.go index 3213b55a8ef..1c918b98fa9 100644 --- a/pkg/api/peer.go +++ b/pkg/api/peer.go @@ -31,7 +31,8 @@ func (s *Service) peerConnectHandler(w http.ResponseWriter, r *http.Request) { return } - bzzAddr, err := s.p2p.Connect(r.Context(), paths.MultiAddress) + bzzAddr, err := s.p2p.Connect(r.Context(), []multiaddr.Multiaddr{paths.MultiAddress}) + if err != nil { logger.Debug("p2p connect failed", "addresses", paths.MultiAddress, "error", err) logger.Error(nil, "p2p connect failed", "addresses", paths.MultiAddress) diff --git a/pkg/api/peer_test.go b/pkg/api/peer_test.go index a9386d8d694..15694632709 100644 --- a/pkg/api/peer_test.go +++ b/pkg/api/peer_test.go @@ -45,15 +45,17 @@ func TestConnect(t *testing.T) { t.Fatal(err) } - bzzAddress, err := bzz.NewAddress(crypto.NewDefaultSigner(privateKey), underlama, overlay, 0, nil) + bzzAddress, err := bzz.NewAddress(crypto.NewDefaultSigner(privateKey), []ma.Multiaddr{underlama}, overlay, 0, nil) if err != nil { t.Fatal(err) } testServer, _, _, _ := newTestServer(t, testServerOptions{ - P2P: mock.New(mock.WithConnectFunc(func(ctx context.Context, addr ma.Multiaddr) (*bzz.Address, error) { - if addr.String() == errorUnderlay { - return nil, testErr + P2P: mock.New(mock.WithConnectFunc(func(ctx context.Context, addrs []ma.Multiaddr) (*bzz.Address, error) { + for _, addr := range addrs { + if addr.String() == errorUnderlay { + return nil, testErr + } } return bzzAddress, nil })), @@ -81,9 +83,11 @@ func TestConnect(t *testing.T) { t.Run("error - add peer", func(t *testing.T) { t.Parallel() testServer, _, _, _ := newTestServer(t, testServerOptions{ - P2P: mock.New(mock.WithConnectFunc(func(ctx context.Context, addr ma.Multiaddr) (*bzz.Address, error) { - if addr.String() == errorUnderlay { - return nil, testErr + P2P: mock.New(mock.WithConnectFunc(func(ctx context.Context, addrs []ma.Multiaddr) (*bzz.Address, error) { + for _, addr := range addrs { + if addr.String() == errorUnderlay { + return nil, testErr + } } return bzzAddress, nil })), diff --git a/pkg/bzz/address.go b/pkg/bzz/address.go index 42fa8fd90b4..1a4021e9eff 100644 --- a/pkg/bzz/address.go +++ b/pkg/bzz/address.go @@ -14,12 +14,15 @@ import ( "encoding/json" "errors" "fmt" + "slices" + "sort" "github.com/ethereum/go-ethereum/common" "github.com/ethersphere/bee/v2/pkg/crypto" "github.com/ethersphere/bee/v2/pkg/swarm" ma "github.com/multiformats/go-multiaddr" + manet "github.com/multiformats/go-multiaddr/net" ) var ErrInvalidAddress = errors.New("invalid address") @@ -28,7 +31,7 @@ var ErrInvalidAddress = errors.New("invalid address") // It consists of a peers underlay (physical) address, overlay (topology) address and signature. // Signature is used to verify the `Overlay/Underlay` pair, as it is based on `underlay|networkID`, signed with the public key of Overlay address type Address struct { - Underlay ma.Multiaddr + Underlays []ma.Multiaddr Overlay swarm.Address Signature []byte Nonce []byte @@ -36,25 +39,23 @@ type Address struct { } type addressJSON struct { - Overlay string `json:"overlay"` - Underlay string `json:"underlay"` - Signature string `json:"signature"` - Nonce string `json:"transaction"` + Overlay string `json:"overlay"` + Underlay string `json:"underlay"` // For backward compatibility + Underlays []string `json:"underlays"` + Signature string `json:"signature"` + Nonce string `json:"transaction"` } -func NewAddress(signer crypto.Signer, underlay ma.Multiaddr, overlay swarm.Address, networkID uint64, nonce []byte) (*Address, error) { - underlayBinary, err := underlay.MarshalBinary() - if err != nil { - return nil, err - } +func NewAddress(signer crypto.Signer, underlays []ma.Multiaddr, overlay swarm.Address, networkID uint64, nonce []byte) (*Address, error) { + underlaysBinary := SerializeUnderlays(underlays) - signature, err := signer.Sign(generateSignData(underlayBinary, overlay.Bytes(), networkID)) + signature, err := signer.Sign(generateSignData(underlaysBinary, overlay.Bytes(), networkID)) if err != nil { return nil, err } return &Address{ - Underlay: underlay, + Underlays: underlays, Overlay: overlay, Signature: signature, Nonce: nonce, @@ -77,8 +78,13 @@ func ParseAddress(underlay, overlay, signature, nonce []byte, validateOverlay bo } } - multiUnderlay, err := ma.NewMultiaddrBytes(underlay) + multiUnderlays, err := DeserializeUnderlays(underlay) if err != nil { + return nil, fmt.Errorf("deserialize underlays: %w: %w", ErrInvalidAddress, err) + } + + if len(multiUnderlays) == 0 { + // no underlays sent return nil, ErrInvalidAddress } @@ -88,7 +94,7 @@ func ParseAddress(underlay, overlay, signature, nonce []byte, validateOverlay bo } return &Address{ - Underlay: multiUnderlay, + Underlays: multiUnderlays, Overlay: swarm.NewAddress(overlay), Signature: signature, Nonce: nonce, @@ -109,21 +115,49 @@ func (a *Address) Equal(b *Address) bool { return a == b } - return a.Overlay.Equal(b.Overlay) && multiaddrEqual(a.Underlay, b.Underlay) && bytes.Equal(a.Signature, b.Signature) && bytes.Equal(a.Nonce, b.Nonce) + return a.Overlay.Equal(b.Overlay) && AreUnderlaysEqual(a.Underlays, b.Underlays) && bytes.Equal(a.Signature, b.Signature) && bytes.Equal(a.Nonce, b.Nonce) } -func multiaddrEqual(a, b ma.Multiaddr) bool { - if a == nil || b == nil { - return a == b +func AreUnderlaysEqual(a, b []ma.Multiaddr) bool { + if len(a) != len(b) { + return false } - return a.Equal(b) + used := make([]bool, len(b)) + for i := range len(a) { + found := false + for j := range len(b) { + if used[j] { + continue + } + if a[i].Equal(b[j]) { + used[j] = true + found = true + break + } + } + if !found { + return false + } + } + return true } func (a *Address) MarshalJSON() ([]byte, error) { + if len(a.Underlays) == 0 { + return nil, fmt.Errorf("no underlays for %s", a.Overlay) + } + + // select the underlay address for backward compatibility + var underlay string + if v := SelectBestAdvertisedAddress(a.Underlays, nil); v != nil { + underlay = v.String() + } + return json.Marshal(&addressJSON{ Overlay: a.Overlay.String(), - Underlay: a.Underlay.String(), + Underlay: underlay, + Underlays: a.underlaysAsStrings(), Signature: base64.StdEncoding.EncodeToString(a.Signature), Nonce: common.Bytes2Hex(a.Nonce), }) @@ -143,23 +177,80 @@ func (a *Address) UnmarshalJSON(b []byte) error { a.Overlay = addr - m, err := ma.NewMultiaddr(v.Underlay) + // append the underlay for backward compatibility + if !slices.Contains(v.Underlays, v.Underlay) { + v.Underlays = append(v.Underlays, v.Underlay) + } + + multiaddrs, err := parseMultiaddrs(v.Underlays) if err != nil { return err } - a.Underlay = m + a.Underlays = multiaddrs a.Signature, err = base64.StdEncoding.DecodeString(v.Signature) a.Nonce = common.Hex2Bytes(v.Nonce) return err } func (a *Address) String() string { - return fmt.Sprintf("[Underlay: %v, Overlay %v, Signature %x, Transaction %x]", a.Underlay, a.Overlay, a.Signature, a.Nonce) + return fmt.Sprintf("[Underlay: %v, Overlay %v, Signature %x, Transaction %x]", a.underlaysAsStrings(), a.Overlay, a.Signature, a.Nonce) } // ShortString returns shortened versions of bzz address in a format: [Overlay, Underlay] // It can be used for logging func (a *Address) ShortString() string { - return fmt.Sprintf("[Overlay: %s, Underlay: %s]", a.Overlay.String(), a.Underlay.String()) + return fmt.Sprintf("[Overlay: %s, Underlays: %v]", a.Overlay.String(), a.underlaysAsStrings()) +} + +func (a *Address) underlaysAsStrings() []string { + underlays := make([]string, len(a.Underlays)) + for i, underlay := range a.Underlays { + underlays[i] = underlay.String() + } + return underlays +} + +func parseMultiaddrs(addrs []string) ([]ma.Multiaddr, error) { + multiAddrs := make([]ma.Multiaddr, len(addrs)) + for i, addr := range addrs { + multiAddr, err := ma.NewMultiaddr(addr) + if err != nil { + return nil, err + } + multiAddrs[i] = multiAddr + } + return multiAddrs, nil +} + +func SelectBestAdvertisedAddress(addrs []ma.Multiaddr, fallback ma.Multiaddr) ma.Multiaddr { + if len(addrs) == 0 { + return fallback + } + + hasTCPProtocol := func(addr ma.Multiaddr) bool { + _, err := addr.ValueForProtocol(ma.P_TCP) + return err == nil + } + + // Sort addresses to prioritize TCP over other protocols + sort.SliceStable(addrs, func(i, j int) bool { + iTCP := hasTCPProtocol(addrs[i]) + jTCP := hasTCPProtocol(addrs[j]) + return iTCP && !jTCP + }) + + for _, addr := range addrs { + if manet.IsPublicAddr(addr) { + return addr + } + } + + for _, addr := range addrs { + if !manet.IsPrivateAddr(addr) { + return addr + } + } + + return addrs[0] } diff --git a/pkg/bzz/address_test.go b/pkg/bzz/address_test.go index 969216d29d7..0c50b08de64 100644 --- a/pkg/bzz/address_test.go +++ b/pkg/bzz/address_test.go @@ -11,13 +11,13 @@ import ( "github.com/ethersphere/bee/v2/pkg/bzz" "github.com/ethersphere/bee/v2/pkg/crypto" - ma "github.com/multiformats/go-multiaddr" + "github.com/multiformats/go-multiaddr" ) func TestBzzAddress(t *testing.T) { t.Parallel() - node1ma, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/1634/p2p/16Uiu2HAkx8ULY8cTXhdVAcMmLcH9AsTKz6uBQ7DPLKRjMLgBVYkA") + node1ma, err := multiaddr.NewMultiaddr("/ip4/127.0.0.1/tcp/1634/p2p/16Uiu2HAkx8ULY8cTXhdVAcMmLcH9AsTKz6uBQ7DPLKRjMLgBVYkA") if err != nil { t.Fatal(err) } @@ -35,7 +35,7 @@ func TestBzzAddress(t *testing.T) { } signer1 := crypto.NewDefaultSigner(privateKey1) - bzzAddress, err := bzz.NewAddress(signer1, node1ma, overlay, 3, nonce) + bzzAddress, err := bzz.NewAddress(signer1, []multiaddr.Multiaddr{node1ma}, overlay, 3, nonce) if err != nil { t.Fatal(err) } @@ -63,3 +63,241 @@ func TestBzzAddress(t *testing.T) { t.Fatalf("got %s expected %s", newbzz, bzzAddress) } } + +func TestSelectBestAdvertisedAddress(t *testing.T) { + t.Parallel() + + mustMultiaddr := func(s string) multiaddr.Multiaddr { + addr, err := multiaddr.NewMultiaddr(s) + if err != nil { + t.Fatalf("failed to create multiaddr %s: %v", s, err) + } + return addr + } + + tests := []struct { + name string + addrs []multiaddr.Multiaddr + fallback multiaddr.Multiaddr + expected multiaddr.Multiaddr + }{ + { + name: "empty addresses returns fallback", + addrs: []multiaddr.Multiaddr{}, + fallback: mustMultiaddr("/ip4/127.0.0.1/tcp/8080"), + expected: mustMultiaddr("/ip4/127.0.0.1/tcp/8080"), + }, + { + name: "nil addresses returns fallback", + addrs: nil, + fallback: mustMultiaddr("/ip4/127.0.0.1/tcp/8080"), + expected: mustMultiaddr("/ip4/127.0.0.1/tcp/8080"), + }, + { + name: "prefers public addresses", + addrs: []multiaddr.Multiaddr{ + mustMultiaddr("/ip4/192.168.1.1/tcp/8080"), // private + mustMultiaddr("/ip4/8.8.8.8/tcp/8080"), // public + mustMultiaddr("/ip4/10.0.0.1/tcp/8080"), // private + }, + fallback: mustMultiaddr("/ip4/127.0.0.1/tcp/8080"), + expected: mustMultiaddr("/ip4/8.8.8.8/tcp/8080"), + }, + { + name: "prefers first public address when multiple exist", + addrs: []multiaddr.Multiaddr{ + mustMultiaddr("/ip4/192.168.1.1/tcp/8080"), // private + mustMultiaddr("/ip4/8.8.8.8/tcp/8080"), // public + mustMultiaddr("/ip4/1.1.1.1/tcp/8080"), // public + }, + fallback: mustMultiaddr("/ip4/127.0.0.1/tcp/8080"), + expected: mustMultiaddr("/ip4/8.8.8.8/tcp/8080"), + }, + { + name: "prefers non-private when no public addresses", + addrs: []multiaddr.Multiaddr{ + mustMultiaddr("/ip4/127.0.0.1/tcp/8080"), // loopback + mustMultiaddr("/ip4/192.168.1.1/tcp/8080"), // private but not loopback + mustMultiaddr("/ip4/10.0.0.1/tcp/8080"), // private but not loopback + }, + fallback: mustMultiaddr("/ip4/127.0.0.1/tcp/9999"), + expected: mustMultiaddr("/ip4/127.0.0.1/tcp/8080"), + }, + { + name: "returns first address when all are loopback", + addrs: []multiaddr.Multiaddr{ + mustMultiaddr("/ip4/127.0.0.1/tcp/8080"), + mustMultiaddr("/ip4/127.0.0.1/tcp/8081"), + mustMultiaddr("/ip6/::1/tcp/8080"), + }, + fallback: mustMultiaddr("/ip4/127.0.0.1/tcp/9999"), + expected: mustMultiaddr("/ip4/127.0.0.1/tcp/8080"), + }, + { + name: "sorts TCP addresses first", + addrs: []multiaddr.Multiaddr{ + mustMultiaddr("/ip4/192.168.1.1/udp/8080"), // UDP + mustMultiaddr("/ip4/1.1.1.1/udp/8080"), // UDP public + mustMultiaddr("/ip4/8.8.8.8/tcp/8080"), // TCP public + }, + fallback: mustMultiaddr("/ip4/127.0.0.1/tcp/9999"), + expected: mustMultiaddr("/ip4/8.8.8.8/tcp/8080"), + }, + { + name: "handles IPv6 addresses", + addrs: []multiaddr.Multiaddr{ + mustMultiaddr("/ip6/::1/tcp/8080"), // loopback + mustMultiaddr("/ip6/2001:db8::1/tcp/8080"), // public IPv6 + mustMultiaddr("/ip4/192.168.1.1/tcp/8080"), // private IPv4 + }, + fallback: mustMultiaddr("/ip4/127.0.0.1/tcp/9999"), + expected: mustMultiaddr("/ip6/2001:db8::1/tcp/8080"), + }, + { + name: "handles mixed protocols with preference order", + addrs: []multiaddr.Multiaddr{ + mustMultiaddr("/ip4/192.168.1.1/udp/8080"), // private UDP + mustMultiaddr("/ip4/192.168.1.2/tcp/8080"), // private TCP + mustMultiaddr("/ip4/127.0.0.1/tcp/8080"), // loopback TCP + }, + fallback: mustMultiaddr("/ip4/127.0.0.1/tcp/9999"), + expected: mustMultiaddr("/ip4/192.168.1.2/tcp/8080"), // first TCP, and it's non-loopback + }, + { + name: "single address", + addrs: []multiaddr.Multiaddr{ + mustMultiaddr("/ip4/192.168.1.1/tcp/8080"), + }, + fallback: mustMultiaddr("/ip4/127.0.0.1/tcp/9999"), + expected: mustMultiaddr("/ip4/192.168.1.1/tcp/8080"), + }, + { + name: "websocket addresses", + addrs: []multiaddr.Multiaddr{ + mustMultiaddr("/ip4/127.0.0.1/tcp/8080/ws"), + mustMultiaddr("/ip4/8.8.8.8/tcp/8080/ws"), // public with websocket + }, + fallback: mustMultiaddr("/ip4/127.0.0.1/tcp/9999"), + expected: mustMultiaddr("/ip4/8.8.8.8/tcp/8080/ws"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := bzz.SelectBestAdvertisedAddress(tt.addrs, tt.fallback) + if !result.Equal(tt.expected) { + t.Errorf("SelectBestAdvertisedAddress() = %v, want %v", result, tt.expected) + } + }) + } +} + +func TestAreUnderlaysEqual(t *testing.T) { + // --- Test Data Initialization --- + addr1 := mustNewMultiaddr(t, "/ip4/127.0.0.1/tcp/8001") + addr2 := mustNewMultiaddr(t, "/ip4/192.168.1.1/tcp/8002") + addr3 := mustNewMultiaddr(t, "/ip6/::1/udp/9000") + addr4 := mustNewMultiaddr(t, "/ip4/127.0.0.1/tcp/8001") // Identical to addr1 + + // --- Test Cases Definition --- + testCases := []struct { + name string + a []multiaddr.Multiaddr + b []multiaddr.Multiaddr + want bool + }{ + { + name: "two nil slices", + a: nil, + b: nil, + want: true, + }, + { + name: "one nil and one empty slice", + a: nil, + b: []multiaddr.Multiaddr{}, + want: true, + }, + { + name: "one empty and one nil slice", + a: []multiaddr.Multiaddr{}, + b: nil, + want: true, + }, + { + name: "two empty slices", + a: []multiaddr.Multiaddr{}, + b: []multiaddr.Multiaddr{}, + want: true, + }, + { + name: "equal slices with same order", + a: []multiaddr.Multiaddr{addr1, addr2}, + b: []multiaddr.Multiaddr{addr1, addr2}, + want: true, + }, + { + name: "equal slices with different order", + a: []multiaddr.Multiaddr{addr1, addr2, addr3}, + b: []multiaddr.Multiaddr{addr3, addr1, addr2}, + want: true, + }, + { + name: "equal slices with identical (but not same instance) values", + a: []multiaddr.Multiaddr{addr1, addr2}, + b: []multiaddr.Multiaddr{addr4, addr2}, + want: true, + }, + { + name: "slices with different lengths (a < b)", + a: []multiaddr.Multiaddr{addr1}, + b: []multiaddr.Multiaddr{addr1, addr2}, + want: false, + }, + { + name: "slices with different lengths (b < a)", + a: []multiaddr.Multiaddr{addr1, addr2}, + b: []multiaddr.Multiaddr{addr1}, + want: false, + }, + { + name: "slices with same length but different elements", + a: []multiaddr.Multiaddr{addr1, addr2}, + b: []multiaddr.Multiaddr{addr1, addr3}, + want: false, + }, + { + name: "one slice is nil", + a: []multiaddr.Multiaddr{addr1}, + b: nil, + want: false, + }, + { + name: "slices with duplicates, equal", + a: []multiaddr.Multiaddr{addr1, addr2, addr1}, + b: []multiaddr.Multiaddr{addr1, addr1, addr2}, + want: true, + }, + { + name: "slices with duplicates, not equal", + a: []multiaddr.Multiaddr{addr1, addr2, addr3}, + b: []multiaddr.Multiaddr{addr1, addr1, addr2}, + want: false, + }, + { + name: "slices with different duplicates", + a: []multiaddr.Multiaddr{addr1, addr1, addr2}, + b: []multiaddr.Multiaddr{addr1, addr2, addr2}, + want: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got := bzz.AreUnderlaysEqual(tc.a, tc.b) + if got != tc.want { + t.Errorf("AreUnderlaysEqual() = %v, want %v", got, tc.want) + } + }) + } +} diff --git a/pkg/bzz/export_test.go b/pkg/bzz/export_test.go new file mode 100644 index 00000000000..531121edc4f --- /dev/null +++ b/pkg/bzz/export_test.go @@ -0,0 +1,7 @@ +// Copyright 2025 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bzz + +const UnderlayListPrefix = underlayListPrefix diff --git a/pkg/bzz/underlay.go b/pkg/bzz/underlay.go new file mode 100644 index 00000000000..fdddf21a404 --- /dev/null +++ b/pkg/bzz/underlay.go @@ -0,0 +1,98 @@ +// Copyright 2025 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bzz + +import ( + "bytes" + "errors" + "fmt" + + "github.com/multiformats/go-multiaddr" + "github.com/multiformats/go-varint" +) + +// underlayListPrefix is a magic byte designated for identifying a serialized list of multiaddrs. +// A value of 0x99 (153) was chosen as it is not a defined multiaddr protocol code. +// This ensures that a failure is triggered by the original multiaddr.NewMultiaddrBytes function, +// which expects a valid protocol code at the start of the data. +const underlayListPrefix byte = 0x99 + +// SerializeUnderlays serializes a slice of multiaddrs into a single byte slice. +// If the slice contains exactly one address, the standard, backward-compatible +// multiaddr format is used. For zero or more than one address, a custom list format +// prefixed with a magic byte is utilized. +func SerializeUnderlays(addrs []multiaddr.Multiaddr) []byte { + // Backward compatibility if exactly one address is present. + if len(addrs) == 1 { + return addrs[0].Bytes() + } + + // For 0 or 2+ addresses, the custom list format with the prefix is used. + // The format is: [prefix_byte][varint_len_1][addr_1_bytes]... + var buf bytes.Buffer + buf.WriteByte(underlayListPrefix) + + for _, addr := range addrs { + addrBytes := addr.Bytes() + buf.Write(varint.ToUvarint(uint64(len(addrBytes)))) + buf.Write(addrBytes) + } + return buf.Bytes() +} + +// DeserializeUnderlays deserializes a byte slice into a slice of multiaddrs. +// The data format is automatically detected as either a single legacy multiaddr +// or a list of multiaddrs (identified by underlayListPrefix), and is parsed accordingly. +func DeserializeUnderlays(data []byte) ([]multiaddr.Multiaddr, error) { + if len(data) == 0 { + return nil, errors.New("cannot deserialize empty byte slice") + } + + // If the data begins with the magic prefix, it is handled as a list. + if data[0] == underlayListPrefix { + return deserializeList(data[1:]) + } + + // Otherwise, the data is handled as a single, backward-compatible multiaddr. + addr, err := multiaddr.NewMultiaddrBytes(data) + if err != nil { + return nil, fmt.Errorf("failed to parse as single multiaddr: %w", err) + } + // The result is returned as a single-element slice for a consistent return type. + return []multiaddr.Multiaddr{addr}, nil +} + +// deserializeList handles the parsing of the custom list format. +// The provided data is expected to have already been stripped of the underlayListPrefix. +func deserializeList(data []byte) ([]multiaddr.Multiaddr, error) { + var addrs []multiaddr.Multiaddr + r := bytes.NewReader(data) + + for r.Len() > 0 { + // The varint-encoded length of the next address is read. + addrLen, err := varint.ReadUvarint(r) + if err != nil { + return nil, fmt.Errorf("failed to read address length from list: %w", err) + } + + // A sanity check is performed to ensure enough bytes remain for the declared length. + if uint64(r.Len()) < addrLen { + return nil, fmt.Errorf("inconsistent data: expected %d bytes for address, but only %d remain", addrLen, r.Len()) + } + + // The individual address bytes are read and parsed. + addrBytes := make([]byte, addrLen) + if _, err := r.Read(addrBytes); err != nil { + return nil, fmt.Errorf("failed to read address bytes: %w", err) + } + + addr, err := multiaddr.NewMultiaddrBytes(addrBytes) + if err != nil { + return nil, fmt.Errorf("failed to parse multiaddr from list: %w", err) + } + addrs = append(addrs, addr) + } + return addrs, nil +} diff --git a/pkg/bzz/underlay_test.go b/pkg/bzz/underlay_test.go new file mode 100644 index 00000000000..f6e4fc7933b --- /dev/null +++ b/pkg/bzz/underlay_test.go @@ -0,0 +1,223 @@ +// Copyright 2025 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bzz_test + +import ( + "bytes" + "reflect" + "testing" + + "github.com/ethersphere/bee/v2/pkg/bzz" + "github.com/multiformats/go-multiaddr" + "github.com/multiformats/go-varint" +) + +func TestSerializeUnderlays(t *testing.T) { + ip4TCPAddr := mustNewMultiaddr(t, "/ip4/127.0.0.1/tcp/80/p2p/QmWqeeHEqG2db37JsuKUxyJ2JF8LtVJMGohKVT8h3aeCVH") + p2pAddr := mustNewMultiaddr(t, "/ip4/65.108.66.216/tcp/16341/p2p/QmVuCJ3M96c7vwv4MQBv7WY1HWQacyCEHvM99R8MUDj95d") + wssAddr := mustNewMultiaddr(t, "/ip4/127.0.0.1/tcp/443/wss/p2p/QmWqeeHEqG2db37JsuKUxyJ2JF8LtVJMGohKVT8h3aeCVH") + dnsSwarmAddr := mustNewMultiaddr(t, "/dnsaddr/mainnet.ethswarm.org") + + t.Run("multiple addresses list", func(t *testing.T) { + addrs := []multiaddr.Multiaddr{ip4TCPAddr, p2pAddr, wssAddr, dnsSwarmAddr} + serialized := bzz.SerializeUnderlays(addrs) + + if serialized[0] != bzz.UnderlayListPrefix { + t.Errorf("expected prefix %x for multiple addresses, got %x", bzz.UnderlayListPrefix, serialized[0]) + } + }) + + t.Run("single address list", func(t *testing.T) { + addrs := []multiaddr.Multiaddr{dnsSwarmAddr} + serialized := bzz.SerializeUnderlays(addrs) + expected := dnsSwarmAddr.Bytes() // Should be legacy format without prefix + + if !bytes.Equal(serialized, expected) { + t.Errorf("expected single address to serialize to legacy format %x, got %x", expected, serialized) + } + if serialized[0] == bzz.UnderlayListPrefix { + t.Error("single address serialization should not have the list prefix") + } + }) + + t.Run("empty list", func(t *testing.T) { + addrs := []multiaddr.Multiaddr{} + serialized := bzz.SerializeUnderlays(addrs) + expected := []byte{bzz.UnderlayListPrefix} + if !bytes.Equal(serialized, expected) { + t.Errorf("expected %x for empty list, got %x", expected, serialized) + } + }) + + t.Run("nil list", func(t *testing.T) { + var addrs []multiaddr.Multiaddr = nil + serialized := bzz.SerializeUnderlays(addrs) + expected := []byte{bzz.UnderlayListPrefix} + if !bytes.Equal(serialized, expected) { + t.Errorf("expected %x for nil list, got %x", expected, serialized) + } + }) +} + +func TestDeserializeUnderlays(t *testing.T) { + ip4TCPAddr := mustNewMultiaddr(t, "/ip4/127.0.0.1/tcp/80/p2p/QmWqeeHEqG2db37JsuKUxyJ2JF8LtVJMGohKVT8h3aeCVH") + p2pAddr := mustNewMultiaddr(t, "/ip4/65.108.66.216/tcp/16341/p2p/QmVuCJ3M96c7vwv4MQBv7WY1HWQacyCEHvM99R8MUDj95d") + wssAddr := mustNewMultiaddr(t, "/ip4/127.0.0.1/tcp/443/wss/p2p/QmWqeeHEqG2db37JsuKUxyJ2JF8LtVJMGohKVT8h3aeCVH") + + t.Run("valid list of multiple", func(t *testing.T) { + addrs := []multiaddr.Multiaddr{ip4TCPAddr, p2pAddr, wssAddr} + serialized := bzz.SerializeUnderlays(addrs) + deserialized, err := bzz.DeserializeUnderlays(serialized) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !reflect.DeepEqual(addrs, deserialized) { + t.Errorf("expected %v, got %v", addrs, deserialized) + } + }) + + t.Run("single legacy multiaddr", func(t *testing.T) { + singleBytes := wssAddr.Bytes() + deserialized, err := bzz.DeserializeUnderlays(singleBytes) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(deserialized) != 1 || !deserialized[0].Equal(wssAddr) { + t.Errorf("expected [%v], got %v", wssAddr, deserialized) + } + }) + + t.Run("empty byte slice", func(t *testing.T) { + _, err := bzz.DeserializeUnderlays([]byte{}) + if err == nil { + t.Error("expected an error for empty slice, but got nil") + } + }) + + t.Run("list with only prefix", func(t *testing.T) { + deserialized, err := bzz.DeserializeUnderlays([]byte{bzz.UnderlayListPrefix}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(deserialized) != 0 { + t.Errorf("expected empty slice, got %v", deserialized) + } + }) + + t.Run("corrupted list - length too long", func(t *testing.T) { + maBytes := ip4TCPAddr.Bytes() + var buf bytes.Buffer + buf.WriteByte(bzz.UnderlayListPrefix) + buf.Write(varint.ToUvarint(uint64(len(maBytes) + 5))) // Write a length that is too long + buf.Write(maBytes) + + _, err := bzz.DeserializeUnderlays(buf.Bytes()) + if err == nil { + t.Error("expected an error for corrupted data, but got nil") + } + }) + + t.Run("corrupted list - invalid multiaddr bytes", func(t *testing.T) { + invalidAddrBytes := []byte{0xde, 0xad, 0xbe, 0xef} + var buf bytes.Buffer + buf.WriteByte(bzz.UnderlayListPrefix) + buf.Write(varint.ToUvarint(uint64(len(invalidAddrBytes)))) + buf.Write(invalidAddrBytes) + + _, err := bzz.DeserializeUnderlays(buf.Bytes()) + if err == nil { + t.Error("expected an error for invalid multiaddr bytes, but got nil") + } + }) +} + +func TestSerializeUnderlaysDeserializeUnderlays(t *testing.T) { + ip4TCPAddr := mustNewMultiaddr(t, "/ip4/127.0.0.1/tcp/80/p2p/QmWqeeHEqG2db37JsuKUxyJ2JF8LtVJMGohKVT8h3aeCVH") + dnsSwarmAddr := mustNewMultiaddr(t, "/dnsaddr/mainnet.ethswarm.org") + p2pAddr := mustNewMultiaddr(t, "/ip4/65.108.66.216/tcp/16341/p2p/QmVuCJ3M96c7vwv4MQBv7WY1HWQacyCEHvM99R8MUDj95d") + wssAddr := mustNewMultiaddr(t, "/ip4/127.0.0.1/tcp/443/wss/p2p/QmWqeeHEqG2db37JsuKUxyJ2JF8LtVJMGohKVT8h3aeCVH") + + t.Run("multiple addresses list", func(t *testing.T) { + addrs := []multiaddr.Multiaddr{ip4TCPAddr, dnsSwarmAddr, p2pAddr, wssAddr} + serialized := bzz.SerializeUnderlays(addrs) + deserialized, err := bzz.DeserializeUnderlays(serialized) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !reflect.DeepEqual(addrs, deserialized) { + t.Errorf("round trip failed. expected %v, got %v", addrs, deserialized) + } + }) + + t.Run("single address list", func(t *testing.T) { + addrs := []multiaddr.Multiaddr{dnsSwarmAddr} + serialized := bzz.SerializeUnderlays(addrs) + deserialized, err := bzz.DeserializeUnderlays(serialized) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !reflect.DeepEqual(addrs, deserialized) { + t.Errorf("round trip failed. expected %v, got %v", addrs, deserialized) + } + }) + + t.Run("empty list", func(t *testing.T) { + addrs := []multiaddr.Multiaddr{} + serialized := bzz.SerializeUnderlays(addrs) + deserialized, err := bzz.DeserializeUnderlays(serialized) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(deserialized) != 0 { + t.Errorf("expected empty slice from round trip, got %v", deserialized) + } + }) +} + +func TestLegacyCompatibility(t *testing.T) { + ip4TCPAddr := mustNewMultiaddr(t, "/ip4/1.2.3.4/tcp/5678/p2p/QmWqeeHEqG2db37JsuKUxyJ2JF8LtVJMGohKVT8h3aeCVH") + p2pAddr := mustNewMultiaddr(t, "/ip4/65.108.66.216/tcp/16341/p2p/QmVuCJ3M96c7vwv4MQBv7WY1HWQacyCEHvM99R8MUDj95d") + dnsSwarmAddr := mustNewMultiaddr(t, "/dnsaddr/mainnet.ethswarm.org") + + t.Run("legacy parser fails on new list format", func(t *testing.T) { + addrs := []multiaddr.Multiaddr{ip4TCPAddr, p2pAddr, dnsSwarmAddr} + listBytes := bzz.SerializeUnderlays(addrs) // This will have the prefix + _, err := multiaddr.NewMultiaddrBytes(listBytes) + if err == nil { + t.Error("expected legacy NewMultiaddrBytes to fail on list format, but it succeeded") + } + }) + + t.Run("legacy parser succeeds on new single-addr format", func(t *testing.T) { + addrs := []multiaddr.Multiaddr{dnsSwarmAddr} + singleBytes := bzz.SerializeUnderlays(addrs) // This will NOT have the prefix + _, err := multiaddr.NewMultiaddrBytes(singleBytes) + if err != nil { + t.Errorf("expected legacy NewMultiaddrBytes to succeed on single-addr format, but it failed: %v", err) + } + }) + + t.Run("new parser succeeds on legacy format", func(t *testing.T) { + singleBytes := p2pAddr.Bytes() + deserialized, err := bzz.DeserializeUnderlays(singleBytes) + if err != nil { + t.Fatalf("Deserialize failed on legacy bytes: %v", err) + } + expected := []multiaddr.Multiaddr{p2pAddr} + if !reflect.DeepEqual(expected, deserialized) { + t.Errorf("expected %v, got %v", expected, deserialized) + } + }) +} + +func mustNewMultiaddr(tb testing.TB, s string) multiaddr.Multiaddr { + tb.Helper() + + a, err := multiaddr.NewMultiaddr(s) + if err != nil { + tb.Fatal(err) + } + return a +} diff --git a/pkg/bzz/utilities_test.go b/pkg/bzz/utilities_test.go index e92e7e11e13..6a9fd9a97dd 100644 --- a/pkg/bzz/utilities_test.go +++ b/pkg/bzz/utilities_test.go @@ -18,6 +18,8 @@ func Test_ContainsAddress(t *testing.T) { t.Parallel() addrs := makeAddreses(t, 10) + multiAddr := makeMultiUnderlayAddress(t) + tt := []struct { addresses []bzz.Address search bzz.Address @@ -32,6 +34,8 @@ func Test_ContainsAddress(t *testing.T) { {addresses: addrs, search: addrs[1], contains: true}, {addresses: addrs, search: addrs[3], contains: true}, {addresses: addrs, search: addrs[9], contains: true}, + {addresses: addrs, search: multiAddr, contains: false}, + {addresses: append(addrs, multiAddr), search: multiAddr, contains: true}, } for _, tc := range tt { @@ -60,7 +64,34 @@ func makeAddress(t *testing.T) bzz.Address { } return bzz.Address{ - Underlay: multiaddr, + Underlays: []ma.Multiaddr{multiaddr}, + Overlay: swarm.RandAddress(t), + Signature: testutil.RandBytes(t, 12), + Nonce: testutil.RandBytes(t, 12), + EthereumAddress: testutil.RandBytes(t, 32), + } +} + +func makeMultiUnderlayAddress(t *testing.T) bzz.Address { + t.Helper() + + addrsStrings := []string{ + "/ip4/127.0.0.1/tcp/1634/p2p/16Uiu2HAkx8ULY8cTXhdVAcMmLcH9AsTKz6uBQ7DPLKRjMLgBVYkA", + "/ip4/10.34.35.60/tcp/35315/p2p/12D3KooWNZujn3N8EcGEFRJrQHw9Hb9v39bNyHC2uw2YNv9jBtjk", + "/ip6/::1/tcp/46881/p2p/12D3KooWNZujn3N8EcGEFRJrQHw9Hb9v39bNyHC2uw2YNv9jBtjk", + } + + underlays := make([]ma.Multiaddr, len(addrsStrings)) + for i, addr := range addrsStrings { + multiaddr, err := ma.NewMultiaddr(addr) + if err != nil { + t.Fatal(err) + } + underlays[i] = multiaddr + } + + return bzz.Address{ + Underlays: underlays, Overlay: swarm.RandAddress(t), Signature: testutil.RandBytes(t, 12), Nonce: testutil.RandBytes(t, 12), diff --git a/pkg/hive/hive.go b/pkg/hive/hive.go index 7932fffc401..f5fe1b3bf53 100644 --- a/pkg/hive/hive.go +++ b/pkg/hive/hive.go @@ -186,13 +186,15 @@ func (s *Service) sendPeers(ctx context.Context, peer swarm.Address, peers []swa return err } - if !s.allowPrivateCIDRs && manet.IsPrivateAddr(addr.Underlay) { - continue // Don't advertise private CIDRs to the public network. + advertisableUnderlays := s.filterAdvertisableUnderlays(addr.Underlays) + if len(advertisableUnderlays) == 0 { + s.logger.Debug("skipping peers: no advertisable underlays", "peer_address", p) + continue } peersRequest.Peers = append(peersRequest.Peers, &pb.BzzAddress{ Overlay: addr.Overlay.Bytes(), - Underlay: addr.Underlay.Bytes(), + Underlay: bzz.SerializeUnderlays(advertisableUnderlays), Signature: addr.Signature, Nonce: addr.Nonce, }) @@ -275,7 +277,7 @@ func (s *Service) checkAndAddPeers(ctx context.Context, peers pb.Peers) { mtx := sync.Mutex{} wg := sync.WaitGroup{} - addPeer := func(newPeer *pb.BzzAddress, multiUnderlay ma.Multiaddr) { + addPeer := func(newPeer *pb.BzzAddress, multiUnderlay []ma.Multiaddr) { err := s.sem.Acquire(ctx, 1) if err != nil { return @@ -293,22 +295,35 @@ func (s *Service) checkAndAddPeers(ctx context.Context, peers pb.Peers) { ctx, cancel := context.WithTimeout(ctx, pingTimeout) defer cancel() - start := time.Now() + var ( + pingSuccessful bool + start time.Time + ) + for _, underlay := range multiUnderlay { + // ping each underlay address, pick first available + start = time.Now() + if _, err := s.streamer.Ping(ctx, underlay); err != nil { + s.logger.Debug("unreachable peer underlay", "peer_address", hex.EncodeToString(newPeer.Overlay), "underlay", underlay, "error", err) + continue + } + pingSuccessful = true + s.logger.Debug("reachable peer underlay", "peer_address", hex.EncodeToString(newPeer.Overlay), "underlay", underlay) + break + } - // check if the underlay is usable by doing a raw ping using libp2p - if _, err := s.streamer.Ping(ctx, multiUnderlay); err != nil { + if !pingSuccessful { + // none of underlay addresses is available s.metrics.PingFailureTime.Observe(time.Since(start).Seconds()) s.metrics.UnreachablePeers.Inc() - s.logger.Debug("unreachable peer underlay", "peer_address", hex.EncodeToString(newPeer.Overlay), "underlay", multiUnderlay) return } - s.metrics.PingTime.Observe(time.Since(start).Seconds()) + s.metrics.PingTime.Observe(time.Since(start).Seconds()) s.metrics.ReachablePeers.Inc() bzzAddress := bzz.Address{ Overlay: swarm.NewAddress(newPeer.Overlay), - Underlay: multiUnderlay, + Underlays: multiUnderlay, Signature: newPeer.Signature, Nonce: newPeer.Nonce, } @@ -327,23 +342,27 @@ func (s *Service) checkAndAddPeers(ctx context.Context, peers pb.Peers) { } for _, p := range peers.Peers { - - multiUnderlay, err := ma.NewMultiaddrBytes(p.Underlay) + multiUnderlays, err := bzz.DeserializeUnderlays(p.Underlay) if err != nil { s.metrics.PeerUnderlayErr.Inc() s.logger.Debug("multi address underlay", "error", err) continue } + if len(multiUnderlays) == 0 { + s.logger.Debug("check and add peers, no underlays", "overlay", swarm.NewAddress(p.Overlay).String()) + continue // no underlays sent + } + // if peer exists already in the addressBook // and if the underlays match, skip addr, err := s.addressBook.Get(swarm.NewAddress(p.Overlay)) - if err == nil && addr.Underlay.Equal(multiUnderlay) { + if err == nil && bzz.AreUnderlaysEqual(addr.Underlays, multiUnderlays) { continue } // add peer does not exist in the addressbook - addPeer(p, multiUnderlay) + addPeer(p, multiUnderlays) } wg.Wait() @@ -351,3 +370,21 @@ func (s *Service) checkAndAddPeers(ctx context.Context, peers pb.Peers) { s.addPeersHandler(peersToAdd...) } } + +// filterAdvertisableUnderlays returns underlay addresses that can be advertised +// based on the allowPrivateCIDRs setting. If allowPrivateCIDRs is false, +// only public addresses are returned. +func (s *Service) filterAdvertisableUnderlays(underlays []ma.Multiaddr) []ma.Multiaddr { + if s.allowPrivateCIDRs { + return underlays + } + + var publicUnderlays []ma.Multiaddr + for _, u := range underlays { + if !manet.IsPrivateAddr(u) { + publicUnderlays = append(publicUnderlays, u) + } + } + + return publicUnderlays +} diff --git a/pkg/hive/hive_test.go b/pkg/hive/hive_test.go index df06c2674ed..1f0d6eecd53 100644 --- a/pkg/hive/hive_test.go +++ b/pkg/hive/hive_test.go @@ -7,6 +7,7 @@ package hive_test import ( "bytes" "context" + "encoding/hex" "errors" "fmt" "strconv" @@ -63,8 +64,12 @@ func TestHandlerRateLimit(t *testing.T) { peers := make([]swarm.Address, hive.LimitBurst+1) for i := range peers { + underlay1, err := ma.NewMultiaddr("/ip4/127.0.0.1/udp/" + strconv.Itoa(i)) + if err != nil { + t.Fatal(err) + } - underlay, err := ma.NewMultiaddr("/ip4/127.0.0.1/udp/" + strconv.Itoa(i)) + underlay2, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/" + strconv.Itoa(i)) if err != nil { t.Fatal(err) } @@ -77,7 +82,7 @@ func TestHandlerRateLimit(t *testing.T) { if err != nil { t.Fatal(err) } - bzzAddr, err := bzz.NewAddress(signer, underlay, overlay, networkID, nonce) + bzzAddr, err := bzz.NewAddress(signer, []ma.Multiaddr{underlay1, underlay2}, overlay, networkID, nonce) if err != nil { t.Fatal(err) } @@ -108,7 +113,6 @@ func TestHandlerRateLimit(t *testing.T) { t.Fatal("want nil error") } } - func TestBroadcastPeers(t *testing.T) { t.Parallel() @@ -127,15 +131,32 @@ func TestBroadcastPeers(t *testing.T) { wantMsgs = append(wantMsgs, pb.Peers{Peers: []*pb.BzzAddress{}}) } + last := 2*hive.MaxBatchSize - 1 + for i := 0; i < 2*hive.MaxBatchSize; i++ { - base := "/ip4/127.0.0.1/udp/" - if i == 2*hive.MaxBatchSize-1 { - base = "/ip4/1.1.1.1/udp/" // The last underlay has public address. - } - underlay, err := ma.NewMultiaddr(base + strconv.Itoa(i)) - if err != nil { - t.Fatal(err) + var underlays []ma.Multiaddr + if i == last { + u, err := ma.NewMultiaddr("/ip4/1.1.1.1/udp/" + strconv.Itoa(i)) // public + if err != nil { + t.Fatal(err) + } + u2, err := ma.NewMultiaddr("/ip4/127.0.0.1/udp/" + strconv.Itoa(i)) + if err != nil { + t.Fatal(err) + } + underlays = []ma.Multiaddr{u, u2} + } else { + n := (i % 3) + 1 + for j := 0; j < n; j++ { + port := i + j*10000 + u, err := ma.NewMultiaddr("/ip4/127.0.0.1/udp/" + strconv.Itoa(port)) + if err != nil { + t.Fatal(err) + } + underlays = append(underlays, u) + } } + pk, err := crypto.GenerateSecp256k1Key() if err != nil { t.Fatal(err) @@ -145,21 +166,20 @@ func TestBroadcastPeers(t *testing.T) { if err != nil { t.Fatal(err) } - bzzAddr, err := bzz.NewAddress(signer, underlay, overlay, networkID, nonce) + bzzAddr, err := bzz.NewAddress(signer, underlays, overlay, networkID, nonce) if err != nil { t.Fatal(err) } bzzAddresses = append(bzzAddresses, *bzzAddr) overlays = append(overlays, bzzAddr.Overlay) - err = addressbook.Put(bzzAddr.Overlay, *bzzAddr) - if err != nil { + if err := addressbook.Put(bzzAddr.Overlay, *bzzAddr); err != nil { t.Fatal(err) } wantMsgs[i/hive.MaxBatchSize].Peers = append(wantMsgs[i/hive.MaxBatchSize].Peers, &pb.BzzAddress{ Overlay: bzzAddresses[i].Overlay.Bytes(), - Underlay: bzzAddresses[i].Underlay.Bytes(), + Underlay: bzz.SerializeUnderlays(bzzAddresses[i].Underlays), Signature: bzzAddresses[i].Signature, Nonce: nonce, }) @@ -223,14 +243,16 @@ func TestBroadcastPeers(t *testing.T) { allowPrivateCIDRs: true, pingErr: func(addr ma.Multiaddr) (rtt time.Duration, err error) { for _, v := range bzzAddresses[10:15] { - if v.Underlay.Equal(addr) { - return rtt, errors.New("ping failure") + for _, underlay := range v.Underlays { + if underlay.Equal(addr) { + return rtt, errors.New("ping failure") + } } } return rtt, nil }, }, - "Ok - don't advertise private CIDRs": { + "Ok - don't advertise private CIDRs only": { addresee: overlays[len(overlays)-1], peers: overlays[:15], wantMsgs: []pb.Peers{{}}, @@ -238,6 +260,29 @@ func TestBroadcastPeers(t *testing.T) { wantBzzAddresses: nil, allowPrivateCIDRs: false, }, + "Ok - don't advertise private CIDRs only (but include one public peer)": { + addresee: overlays[0], + peers: overlays[58:], + wantMsgs: []pb.Peers{{Peers: []*pb.BzzAddress{ + { + Overlay: bzzAddresses[len(bzzAddresses)-1].Overlay.Bytes(), + Underlay: bzz.SerializeUnderlays([]ma.Multiaddr{bzzAddresses[len(bzzAddresses)-1].Underlays[0]}), + Signature: bzzAddresses[len(bzzAddresses)-1].Signature, + Nonce: nonce, + }, + }}}, + wantOverlays: []swarm.Address{overlays[len(overlays)-1]}, + wantBzzAddresses: []bzz.Address{ + { + Underlays: []ma.Multiaddr{bzzAddresses[len(bzzAddresses)-1].Underlays[0]}, + Overlay: bzzAddresses[len(bzzAddresses)-1].Overlay, + Signature: bzzAddresses[len(bzzAddresses)-1].Signature, + Nonce: bzzAddresses[len(bzzAddresses)-1].Nonce, + EthereumAddress: bzzAddresses[len(bzzAddresses)-1].EthereumAddress, + }, + }, + allowPrivateCIDRs: false, + }, } for name, tc := range testCases { @@ -264,6 +309,7 @@ func TestBroadcastPeers(t *testing.T) { // create a hive client that will do broadcast client := hive.New(recorder, addressbook, networkID, false, tc.allowPrivateCIDRs, logger) + if err := client.BroadcastPeers(context.Background(), tc.addresee, tc.peers...); err != nil { t.Fatal(err) } @@ -284,10 +330,7 @@ func TestBroadcastPeers(t *testing.T) { if err != nil { t.Fatal(err) } - - if fmt.Sprint(messages[0]) != fmt.Sprint(tc.wantMsgs[i]) { - t.Errorf("Messages got %v, want %v", messages, tc.wantMsgs) - } + comparePeerMsgs(t, messages[0].Peers, tc.wantMsgs[i].Peers) } expectOverlaysEventually(t, addressbookclean, tc.wantOverlays) @@ -380,3 +423,50 @@ func readAndAssertPeersMsgs(in []byte, expectedLen int) ([]pb.Peers, error) { return peers, nil } + +func comparePeerMsgs(t *testing.T, got, want []*pb.BzzAddress) { + t.Helper() + + gotMap := flattenByOverlay(t, got) + wantMap := flattenByOverlay(t, want) + + for ovlHex, w := range wantMap { + g, ok := gotMap[ovlHex] + if !ok { + t.Fatalf("expected peer %s, but not found", ovlHex) + } + if !bytes.Equal(g.Underlay, w.Underlay) { + t.Fatalf("peer %s: underlays (got=%s want=%s)", ovlHex, g.Underlay, w.Underlay) + } + if !bytes.Equal(g.Signature, w.Signature) { + t.Fatalf("peer %s: expected signatures (got=%s want=%s)", + ovlHex, shortHex(g.Signature), shortHex(w.Signature)) + } + if !bytes.Equal(g.Nonce, w.Nonce) { + t.Fatalf("peer %s: expected nonce (got=%s want=%s)", + ovlHex, shortHex(g.Nonce), shortHex(w.Nonce)) + } + } +} + +func flattenByOverlay(t *testing.T, batches []*pb.BzzAddress) map[string]*pb.BzzAddress { + t.Helper() + + m := make(map[string]*pb.BzzAddress) + for _, batch := range batches { + overlay := hex.EncodeToString(batch.Overlay) + if _, ok := m[overlay]; ok { + t.Fatalf("multiple bzz addresses with the same overlay address: %s", overlay) + } + m[overlay] = batch + } + return m +} + +func shortHex(b []byte) string { + s := hex.EncodeToString(b) + if len(s) > 32 { + return s[:32] + fmt.Sprintf("…(%dB)", len(b)) + } + return s +} diff --git a/pkg/node/devnode.go b/pkg/node/devnode.go index acf69ce026f..b3b7e1e8677 100644 --- a/pkg/node/devnode.go +++ b/pkg/node/devnode.go @@ -244,7 +244,7 @@ func NewDevBee(logger log.Logger, o *DevOptions) (b *DevBee, err error) { lightNodes = lightnode.NewContainer(swarm.NewAddress(nil)) pingPong = mockPingPong.New(pong) p2ps = mockP2P.New( - mockP2P.WithConnectFunc(func(ctx context.Context, addr multiaddr.Multiaddr) (address *bzz.Address, err error) { + mockP2P.WithConnectFunc(func(ctx context.Context, addr []multiaddr.Multiaddr) (address *bzz.Address, err error) { return &bzz.Address{}, nil }), mockP2P.WithDisconnectFunc( func(swarm.Address, string) error { diff --git a/pkg/node/node.go b/pkg/node/node.go index d5ba02bdc56..0a4d526803c 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -1163,7 +1163,7 @@ func NewBee( logger.Debug("sync status check", "synced", synced, "reserveSize", localStore.ReserveSize(), "syncRate", pullerService.SyncRate()) if synced { fullSyncTime := pullSyncStartTime.Sub(t) - logger.Info("full sync duration", "duration", fullSyncTime) + logger.Info("full sync done", "duration", fullSyncTime) nodeMetrics.FullSyncDuration.Observe(fullSyncTime.Minutes()) syncCheckTicker.Stop() return diff --git a/pkg/p2p/libp2p/connections_test.go b/pkg/p2p/libp2p/connections_test.go index 93d8ab41740..42187edca9a 100644 --- a/pkg/p2p/libp2p/connections_test.go +++ b/pkg/p2p/libp2p/connections_test.go @@ -16,6 +16,8 @@ import ( "testing" "time" + "github.com/ethersphere/bee/v2/pkg/bzz" + "github.com/ethersphere/bee/v2/pkg/addressbook" "github.com/ethersphere/bee/v2/pkg/log" "github.com/ethersphere/bee/v2/pkg/p2p" @@ -398,7 +400,7 @@ func TestConnectDisconnectOnAllAddresses(t *testing.T) { t.Fatal(err) } for _, addr := range addrs { - bzzAddr, err := s2.Connect(ctx, addr) + bzzAddr, err := s2.Connect(ctx, []ma.Multiaddr{addr}) if err != nil { t.Fatal(err) } @@ -434,14 +436,14 @@ func TestDoubleConnectOnAllAddresses(t *testing.T) { // creating new remote host for each address s2, overlay2 := newService(t, 1, libp2pServiceOpts{notifier: mockNotifier(noopCf, noopDf, true)}) - if _, err := s2.Connect(ctx, addr); err != nil { + if _, err := s2.Connect(ctx, []ma.Multiaddr{addr}); err != nil { t.Fatal(err) } expectPeers(t, s2, overlay1) expectPeersEventually(t, s1, overlay2) - if _, err := s2.Connect(ctx, addr); !errors.Is(err, p2p.ErrAlreadyConnected) { + if _, err := s2.Connect(ctx, []ma.Multiaddr{addr}); !errors.Is(err, p2p.ErrAlreadyConnected) { t.Fatalf("expected %s error, got %s error", p2p.ErrAlreadyConnected, err) } @@ -517,9 +519,8 @@ func TestConnectRepeatHandshake(t *testing.T) { }}) s2, overlay2 := newService(t, 1, libp2pServiceOpts{}) - addr := serviceUnderlayAddress(t, s1) - - _, err := s2.Connect(ctx, addr) + addrs := serviceUnderlayAddress(t, s1) + _, err := s2.Connect(ctx, addrs) if err != nil { t.Fatal(err) } @@ -527,7 +528,7 @@ func TestConnectRepeatHandshake(t *testing.T) { expectPeers(t, s2, overlay1) expectPeersEventually(t, s1, overlay2) - info, err := libp2ppeer.AddrInfoFromP2pAddr(addr) + info, err := libp2ppeer.AddrInfoFromP2pAddr(addrs[0]) if err != nil { t.Fatal(err) } @@ -537,7 +538,7 @@ func TestConnectRepeatHandshake(t *testing.T) { t.Fatal(err) } - if _, err := s2.HandshakeService().Handshake(ctx, s2.WrapStream(stream), info.Addrs[0], info.ID); err != nil { + if _, err := s2.HandshakeService().Handshake(ctx, s2.WrapStream(stream), addrs); err != nil { t.Fatal(err) } @@ -622,8 +623,8 @@ func TestBlocklistedPeers(t *testing.T) { FullNode: true, }}) s2, _ := newService(t, 1, libp2pServiceOpts{}) - addr1 := serviceUnderlayAddress(t, s1) - _, err := s2.Connect(context.Background(), addr1) + addrs1 := serviceUnderlayAddress(t, s1) + _, err := s2.Connect(context.Background(), addrs1) if err != nil { t.Fatal(err) } @@ -714,10 +715,10 @@ func TestTopologyNotifier(t *testing.T) { }) s2.SetPickyNotifier(notifier2) - addr := serviceUnderlayAddress(t, s1) + s1Addr := serviceUnderlayAddress(t, s1) // s2 connects to s1, thus the notifier on s1 should be called on Connect - bzzAddr, err := s2.Connect(ctx, addr) + bzzAddr, err := s2.Connect(ctx, s1Addr) if err != nil { t.Fatal(err) } @@ -733,7 +734,7 @@ func TestTopologyNotifier(t *testing.T) { mtx.Unlock() // check address book entries are there - checkAddressbook(t, ab2, overlay1, addr) + checkAddressbook(t, ab2, overlay1, s1Addr) // s2 disconnects from s1 so s1 disconnect notifiee should be called if err := s2.Disconnect(bzzAddr.Overlay, testDisconnectMsg); err != nil { @@ -1256,7 +1257,7 @@ func waitAddrSet(t *testing.T, addr *swarm.Address, mtx *sync.Mutex, exp swarm.A } } -func checkAddressbook(t *testing.T, ab addressbook.Getter, overlay swarm.Address, underlay ma.Multiaddr) { +func checkAddressbook(t *testing.T, ab addressbook.Getter, overlay swarm.Address, underlays []ma.Multiaddr) { t.Helper() addr, err := ab.Get(overlay) if err != nil { @@ -1266,8 +1267,8 @@ func checkAddressbook(t *testing.T, ab addressbook.Getter, overlay swarm.Address t.Fatalf("overlay mismatch. got %s want %s", addr.Overlay, overlay) } - if !addr.Underlay.Equal(underlay) { - t.Fatalf("underlay mismatch. got %s, want %s", addr.Underlay, underlay) + if !bzz.AreUnderlaysEqual(addr.Underlays, underlays) { + t.Fatalf("underlay mismatch. got %s, want %s", addr.Underlays, underlays) } } diff --git a/pkg/p2p/libp2p/internal/handshake/handshake.go b/pkg/p2p/libp2p/internal/handshake/handshake.go index cb478c71d4a..0dcc726d7d7 100644 --- a/pkg/p2p/libp2p/internal/handshake/handshake.go +++ b/pkg/p2p/libp2p/internal/handshake/handshake.go @@ -5,9 +5,11 @@ package handshake import ( + "cmp" "context" "errors" "fmt" + "slices" "sync/atomic" "time" @@ -60,6 +62,24 @@ type AdvertisableAddressResolver interface { Resolve(observedAddress ma.Multiaddr) (ma.Multiaddr, error) } +type Addresser interface { + AdvertizableAddrs() ([]ma.Multiaddr, error) +} + +type Option struct { + bee260compatibility bool +} + +// WithBee260Compatibility option ensures that only one underlay address is +// passed to the peer in p2p protocol messages, so that nodes with version 2.6.0 +// and older can deserialize it. This option can be safely removed when bee +// version 2.6.0 is deprecated. +func WithBee260Compatibility(yes bool) func(*Option) { + return func(o *Option) { + o.bee260compatibility = yes + } +} + // Service can perform initiate or handle a handshake between peers. type Service struct { signer crypto.Signer @@ -74,6 +94,7 @@ type Service struct { libp2pID libp2ppeer.ID metrics metrics picker p2p.Picker + hostAddresser Addresser } // Info contains the information received from the handshake. @@ -91,7 +112,7 @@ func (i *Info) LightString() string { } // New creates a new handshake Service. -func New(signer crypto.Signer, advertisableAddresser AdvertisableAddressResolver, overlay swarm.Address, networkID uint64, fullNode bool, nonce []byte, welcomeMessage string, validateOverlay bool, ownPeerID libp2ppeer.ID, logger log.Logger) (*Service, error) { +func New(signer crypto.Signer, advertisableAddresser AdvertisableAddressResolver, overlay swarm.Address, networkID uint64, fullNode bool, nonce []byte, hostAddresser Addresser, welcomeMessage string, validateOverlay bool, ownPeerID libp2ppeer.ID, logger log.Logger) (*Service, error) { if len(welcomeMessage) > MaxWelcomeMessageLength { return nil, ErrWelcomeMessageLength } @@ -107,6 +128,7 @@ func New(signer crypto.Signer, advertisableAddresser AdvertisableAddressResolver libp2pID: ownPeerID, logger: logger.WithName(loggerName).Register(), metrics: newMetrics(), + hostAddresser: hostAddresser, } svc.welcomeMessage.Store(welcomeMessage) @@ -118,25 +140,23 @@ func (s *Service) SetPicker(n p2p.Picker) { } // Handshake initiates a handshake with a peer. -func (s *Service) Handshake(ctx context.Context, stream p2p.Stream, peerMultiaddr ma.Multiaddr, peerID libp2ppeer.ID) (i *Info, err error) { +func (s *Service) Handshake(ctx context.Context, stream p2p.Stream, peerMultiaddrs []ma.Multiaddr, opts ...func(*Option)) (i *Info, err error) { loggerV1 := s.logger.V(1).Register() + o := new(Option) + for _, set := range opts { + set(o) + } + ctx, cancel := context.WithTimeout(ctx, handshakeTimeout) defer cancel() w, r := protobuf.NewWriterAndReader(stream) - fullRemoteMA, err := buildFullMA(peerMultiaddr, peerID) - if err != nil { - return nil, err - } - fullRemoteMABytes, err := fullRemoteMA.MarshalBinary() - if err != nil { - return nil, err - } + peerMultiaddrs = filterBee260CompatibleUnderlays(o.bee260compatibility, peerMultiaddrs) if err := w.WriteMsgWithContext(ctx, &pb.Syn{ - ObservedUnderlay: fullRemoteMABytes, + ObservedUnderlay: bzz.SerializeUnderlays(peerMultiaddrs), }); err != nil { return nil, fmt.Errorf("write syn message: %w", err) } @@ -146,32 +166,51 @@ func (s *Service) Handshake(ctx context.Context, stream p2p.Stream, peerMultiadd return nil, fmt.Errorf("read synack message: %w", err) } - observedUnderlay, err := ma.NewMultiaddrBytes(resp.Syn.ObservedUnderlay) + observedUnderlays, err := bzz.DeserializeUnderlays(resp.Syn.ObservedUnderlay) if err != nil { return nil, ErrInvalidSyn } - observedUnderlayAddrInfo, err := libp2ppeer.AddrInfoFromP2pAddr(observedUnderlay) - if err != nil { - return nil, fmt.Errorf("extract addr from P2P: %w", err) - } + advertisableUnderlays := make([]ma.Multiaddr, len(observedUnderlays)) + for i, observedUnderlay := range observedUnderlays { + observedUnderlayAddrInfo, err := libp2ppeer.AddrInfoFromP2pAddr(observedUnderlay) + if err != nil { + return nil, fmt.Errorf("extract addr from P2P: %w", err) + } - if s.libp2pID != observedUnderlayAddrInfo.ID { - // NOTE eventually we will return error here, but for now we want to gather some statistics - s.logger.Warning("received peer ID does not match ours", "their", observedUnderlayAddrInfo.ID, "ours", s.libp2pID) - } + if s.libp2pID != observedUnderlayAddrInfo.ID { + return nil, fmt.Errorf("received peer ID %s does not match ours %s", observedUnderlayAddrInfo.ID.String(), s.libp2pID.String()) + } - advertisableUnderlay, err := s.advertisableAddresser.Resolve(observedUnderlay) - if err != nil { - return nil, err + advertisableUnderlay, err := s.advertisableAddresser.Resolve(observedUnderlay) + if err != nil { + return nil, err + } + + advertisableUnderlays[i] = advertisableUnderlay } - bzzAddress, err := bzz.NewAddress(s.signer, advertisableUnderlay, s.overlay, s.networkID, s.nonce) - if err != nil { - return nil, err + if s.hostAddresser != nil { + hostAddrs, err := s.hostAddresser.AdvertizableAddrs() + if err != nil { + return nil, fmt.Errorf("get host advertizable addresses: %w", err) + } + + advertisableUnderlays = append(advertisableUnderlays, hostAddrs...) } - advertisableUnderlayBytes, err := bzzAddress.Underlay.MarshalBinary() + // sort to remove potential duplicates + slices.SortFunc(advertisableUnderlays, func(a, b ma.Multiaddr) int { + return cmp.Compare(a.String(), b.String()) + }) + // remove duplicates + advertisableUnderlays = slices.CompactFunc(advertisableUnderlays, func(a, b ma.Multiaddr) bool { + return a.Equal(b) + }) + + advertisableUnderlays = filterBee260CompatibleUnderlays(o.bee260compatibility, advertisableUnderlays) + + bzzAddress, err := bzz.NewAddress(s.signer, advertisableUnderlays, s.overlay, s.networkID, s.nonce) if err != nil { return nil, err } @@ -189,7 +228,7 @@ func (s *Service) Handshake(ctx context.Context, stream p2p.Stream, peerMultiadd welcomeMessage := s.GetWelcomeMessage() msg := &pb.Ack{ Address: &pb.BzzAddress{ - Underlay: advertisableUnderlayBytes, + Underlay: bzz.SerializeUnderlays(bzzAddress.Underlays), Overlay: bzzAddress.Overlay.Bytes(), Signature: bzzAddress.Signature, }, @@ -215,22 +254,18 @@ func (s *Service) Handshake(ctx context.Context, stream p2p.Stream, peerMultiadd } // Handle handles an incoming handshake from a peer. -func (s *Service) Handle(ctx context.Context, stream p2p.Stream, remoteMultiaddr ma.Multiaddr, remotePeerID libp2ppeer.ID) (i *Info, err error) { +func (s *Service) Handle(ctx context.Context, stream p2p.Stream, peerMultiaddrs []ma.Multiaddr, opts ...func(*Option)) (i *Info, err error) { loggerV1 := s.logger.V(1).Register() + o := new(Option) + for _, set := range opts { + set(o) + } + ctx, cancel := context.WithTimeout(ctx, handshakeTimeout) defer cancel() w, r := protobuf.NewWriterAndReader(stream) - fullRemoteMA, err := buildFullMA(remoteMultiaddr, remotePeerID) - if err != nil { - return nil, err - } - - fullRemoteMABytes, err := fullRemoteMA.MarshalBinary() - if err != nil { - return nil, err - } var syn pb.Syn if err := r.ReadMsgWithContext(ctx, &syn); err != nil { @@ -239,35 +274,56 @@ func (s *Service) Handle(ctx context.Context, stream p2p.Stream, remoteMultiaddr } s.metrics.SynRx.Inc() - observedUnderlay, err := ma.NewMultiaddrBytes(syn.ObservedUnderlay) + observedUnderlays, err := bzz.DeserializeUnderlays(syn.ObservedUnderlay) if err != nil { return nil, ErrInvalidSyn } - advertisableUnderlay, err := s.advertisableAddresser.Resolve(observedUnderlay) - if err != nil { - return nil, err + advertisableUnderlays := make([]ma.Multiaddr, len(observedUnderlays)) + for i, observedUnderlay := range observedUnderlays { + advertisableUnderlay, err := s.advertisableAddresser.Resolve(observedUnderlay) + if err != nil { + return nil, err + } + advertisableUnderlays[i] = advertisableUnderlay } - bzzAddress, err := bzz.NewAddress(s.signer, advertisableUnderlay, s.overlay, s.networkID, s.nonce) - if err != nil { - return nil, err + if s.hostAddresser != nil { + hostAddrs, err := s.hostAddresser.AdvertizableAddrs() + if err != nil { + return nil, fmt.Errorf("get host advertizable addresses: %w", err) + } + + advertisableUnderlays = append(advertisableUnderlays, hostAddrs...) } - advertisableUnderlayBytes, err := bzzAddress.Underlay.MarshalBinary() + // sort to remove potential duplicates + slices.SortFunc(advertisableUnderlays, func(a, b ma.Multiaddr) int { + return cmp.Compare(a.String(), b.String()) + }) + // remove duplicates + advertisableUnderlays = slices.CompactFunc(advertisableUnderlays, func(a, b ma.Multiaddr) bool { + return a.Equal(b) + }) + + advertisableUnderlays = filterBee260CompatibleUnderlays(o.bee260compatibility, advertisableUnderlays) + + bzzAddress, err := bzz.NewAddress(s.signer, advertisableUnderlays, s.overlay, s.networkID, s.nonce) if err != nil { return nil, err } welcomeMessage := s.GetWelcomeMessage() + peerMultiaddrs = filterBee260CompatibleUnderlays(o.bee260compatibility, peerMultiaddrs) + if err := w.WriteMsgWithContext(ctx, &pb.SynAck{ Syn: &pb.Syn{ - ObservedUnderlay: fullRemoteMABytes, + ObservedUnderlay: bzz.SerializeUnderlays(peerMultiaddrs), }, Ack: &pb.Ack{ Address: &pb.BzzAddress{ - Underlay: advertisableUnderlayBytes, + Underlay: bzz.SerializeUnderlays(bzzAddress.Underlays), Overlay: bzzAddress.Overlay.Bytes(), Signature: bzzAddress.Signature, }, @@ -331,10 +387,6 @@ func (s *Service) GetWelcomeMessage() string { return s.welcomeMessage.Load().(string) } -func buildFullMA(addr ma.Multiaddr, peerID libp2ppeer.ID) (ma.Multiaddr, error) { - return ma.NewMultiaddr(fmt.Sprintf("%s/p2p/%s", addr.String(), peerID.String())) -} - func (s *Service) parseCheckAck(ack *pb.Ack) (*bzz.Address, error) { bzzAddress, err := bzz.ParseAddress(ack.Address.Underlay, ack.Address.Overlay, ack.Address.Signature, ack.Nonce, s.validateOverlay, s.networkID) if err != nil { @@ -343,3 +395,18 @@ func (s *Service) parseCheckAck(ack *pb.Ack) (*bzz.Address, error) { return bzzAddress, nil } + +// filterBee260CompatibleUnderlays select a single underlay to pass if +// bee260compatibility is true. Otherwise it passes the unmodified underlays +// slice. This function can be safely removed when bee version 2.6.0 is +// deprecated. +func filterBee260CompatibleUnderlays(bee260compatibility bool, underlays []ma.Multiaddr) []ma.Multiaddr { + if !bee260compatibility { + return underlays + } + underlay := bzz.SelectBestAdvertisedAddress(underlays, nil) + if underlay == nil { + return underlays + } + return []ma.Multiaddr{underlay} +} diff --git a/pkg/p2p/libp2p/internal/handshake/handshake_test.go b/pkg/p2p/libp2p/internal/handshake/handshake_test.go index e0afcc56d70..d455e6e8520 100644 --- a/pkg/p2p/libp2p/internal/handshake/handshake_test.go +++ b/pkg/p2p/libp2p/internal/handshake/handshake_test.go @@ -38,27 +38,44 @@ func TestHandshake(t *testing.T) { if err != nil { t.Fatal(err) } - node2ma, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/1634/p2p/16Uiu2HAkx8ULY8cTXhdVAcMmLcH9AsTKz6uBQ7DPLKRjMLgBVYkS") + + node1ma2, err := ma.NewMultiaddr("/ip6/::1/tcp/46881/p2p/16Uiu2HAkx8ULY8cTXhdVAcMmLcH9AsTKz6uBQ7DPLKRjMLgBVYkA") if err != nil { t.Fatal(err) } - node1maBinary, err := node1ma.MarshalBinary() + + node2ma, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/1634/p2p/16Uiu2HAkx8ULY8cTXhdVAcMmLcH9AsTKz6uBQ7DPLKRjMLgBVYkS") if err != nil { t.Fatal(err) } - node2maBinary, err := node2ma.MarshalBinary() + + node2ma2, err := ma.NewMultiaddr("/ip4/10.34.35.60/tcp/35315/p2p/16Uiu2HAkx8ULY8cTXhdVAcMmLcH9AsTKz6uBQ7DPLKRjMLgBVYkS") if err != nil { t.Fatal(err) } - node1AddrInfo, err := libp2ppeer.AddrInfoFromP2pAddr(node1ma) + + node1mas := []ma.Multiaddr{node1ma, node1ma2} + node2mas := []ma.Multiaddr{node2ma, node2ma2} + + node1maBinary := bzz.SerializeUnderlays(node1mas) + node2maBinary := bzz.SerializeUnderlays(node2mas) + + node1AddrInfos, err := libp2ppeer.AddrInfosFromP2pAddrs(node1ma, node1ma2) if err != nil { t.Fatal(err) } - node2AddrInfo, err := libp2ppeer.AddrInfoFromP2pAddr(node2ma) + if len(node1AddrInfos) != 1 { + t.Fatal("must be same peer") + } + node1AddrInfo := node1AddrInfos[0] + + node2AddrInfos, err := libp2ppeer.AddrInfosFromP2pAddrs(node2ma, node2ma2) if err != nil { t.Fatal(err) } - + if len(node2AddrInfos) != 1 { + t.Fatal("must be same peer") + } privateKey1, err := crypto.GenerateSecp256k1Key() if err != nil { t.Fatal(err) @@ -76,7 +93,7 @@ func TestHandshake(t *testing.T) { if err != nil { t.Fatal(err) } - node1BzzAddress, err := bzz.NewAddress(signer1, node1ma, addr, networkID, nonce) + node1BzzAddress, err := bzz.NewAddress(signer1, []ma.Multiaddr{node1ma, node1ma2}, addr, networkID, nonce) if err != nil { t.Fatal(err) } @@ -84,7 +101,7 @@ func TestHandshake(t *testing.T) { if err != nil { t.Fatal(err) } - node2BzzAddress, err := bzz.NewAddress(signer2, node2ma, addr2, networkID, nonce) + node2BzzAddress, err := bzz.NewAddress(signer2, []ma.Multiaddr{node2ma, node2ma2}, addr2, networkID, nonce) if err != nil { t.Fatal(err) } @@ -100,7 +117,7 @@ func TestHandshake(t *testing.T) { aaddresser := &AdvertisableAddresserMock{} - handshakeService, err := handshake.New(signer1, aaddresser, node1Info.BzzAddress.Overlay, networkID, true, nonce, testWelcomeMessage, true, node1AddrInfo.ID, logger) + handshakeService, err := handshake.New(signer1, aaddresser, node1Info.BzzAddress.Overlay, networkID, true, nonce, nil, testWelcomeMessage, true, node1AddrInfo.ID, logger) if err != nil { t.Fatal(err) } @@ -134,7 +151,7 @@ func TestHandshake(t *testing.T) { t.Fatal(err) } - res, err := handshakeService.Handshake(context.Background(), stream1, node2AddrInfo.Addrs[0], node2AddrInfo.ID) + res, err := handshakeService.Handshake(context.Background(), stream1, node2mas) if err != nil { t.Fatal(err) } @@ -177,7 +194,7 @@ func TestHandshake(t *testing.T) { }) t.Run("Handshake - picker error", func(t *testing.T) { - handshakeService, err := handshake.New(signer1, aaddresser, node1Info.BzzAddress.Overlay, networkID, true, nonce, "", true, node1AddrInfo.ID, logger) + handshakeService, err := handshake.New(signer1, aaddresser, node1Info.BzzAddress.Overlay, networkID, true, nonce, nil, "", true, node1AddrInfo.ID, logger) if err != nil { t.Fatal(err) } @@ -209,7 +226,7 @@ func TestHandshake(t *testing.T) { t.Fatal(err) } - _, err = handshakeService.Handle(context.Background(), stream1, node2AddrInfo.Addrs[0], node2AddrInfo.ID) + _, err = handshakeService.Handle(context.Background(), stream1, node2mas) expectedErr := handshake.ErrPicker if !errors.Is(err, expectedErr) { t.Fatal("expected:", expectedErr, "got:", err) @@ -220,7 +237,7 @@ func TestHandshake(t *testing.T) { const LongMessage = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Morbi consectetur urna ut lorem sollicitudin posuere. Donec sagittis laoreet sapien." expectedErr := handshake.ErrWelcomeMessageLength - _, err := handshake.New(signer1, aaddresser, node1Info.BzzAddress.Overlay, networkID, true, nil, LongMessage, true, node1AddrInfo.ID, logger) + _, err := handshake.New(signer1, aaddresser, node1Info.BzzAddress.Overlay, networkID, true, nil, nil, LongMessage, true, node1AddrInfo.ID, logger) if err == nil || err.Error() != expectedErr.Error() { t.Fatal("expected:", expectedErr, "got:", err) } @@ -254,7 +271,7 @@ func TestHandshake(t *testing.T) { expectedErr := fmt.Errorf("write syn message: %w", testErr) stream := &mock.Stream{} stream.SetWriteErr(testErr, 0) - res, err := handshakeService.Handshake(context.Background(), stream, node2AddrInfo.Addrs[0], node2AddrInfo.ID) + res, err := handshakeService.Handshake(context.Background(), stream, node2mas) if err == nil || err.Error() != expectedErr.Error() { t.Fatal("expected:", expectedErr, "got:", err) } @@ -269,7 +286,7 @@ func TestHandshake(t *testing.T) { expectedErr := fmt.Errorf("read synack message: %w", testErr) stream := mock.NewStream(nil, &bytes.Buffer{}) stream.SetReadErr(testErr, 0) - res, err := handshakeService.Handshake(context.Background(), stream, node2AddrInfo.Addrs[0], node2AddrInfo.ID) + res, err := handshakeService.Handshake(context.Background(), stream, node2mas) if err == nil || err.Error() != expectedErr.Error() { t.Fatal("expected:", expectedErr, "got:", err) } @@ -308,7 +325,7 @@ func TestHandshake(t *testing.T) { t.Fatal(err) } - res, err := handshakeService.Handshake(context.Background(), stream1, node2AddrInfo.Addrs[0], node2AddrInfo.ID) + res, err := handshakeService.Handshake(context.Background(), stream1, node2mas) if err == nil || err.Error() != expectedErr.Error() { t.Fatal("expected:", expectedErr, "got:", err) } @@ -342,7 +359,7 @@ func TestHandshake(t *testing.T) { t.Fatal(err) } - res, err := handshakeService.Handshake(context.Background(), stream1, node2AddrInfo.Addrs[0], node2AddrInfo.ID) + res, err := handshakeService.Handshake(context.Background(), stream1, node2mas) if res != nil { t.Fatal("res should be nil") } @@ -375,11 +392,11 @@ func TestHandshake(t *testing.T) { }); err != nil { t.Fatal(err) } - handshakeService, err := handshake.New(signer1, aaddresser, node1Info.BzzAddress.Overlay, networkID, true, nonce, testWelcomeMessage, true, node1AddrInfo.ID, logger) + handshakeService, err := handshake.New(signer1, aaddresser, node1Info.BzzAddress.Overlay, networkID, true, nonce, nil, testWelcomeMessage, true, node1AddrInfo.ID, logger) if err != nil { t.Fatal(err) } - res, err := handshakeService.Handshake(context.Background(), stream1, node2AddrInfo.Addrs[0], node2AddrInfo.ID) + res, err := handshakeService.Handshake(context.Background(), stream1, node2mas) if res != nil { t.Fatal("res should be nil") } @@ -419,7 +436,7 @@ func TestHandshake(t *testing.T) { t.Fatal(err) } - res, err := handshakeService.Handshake(context.Background(), stream1, node2AddrInfo.Addrs[0], node2AddrInfo.ID) + res, err := handshakeService.Handshake(context.Background(), stream1, node2mas) if !errors.Is(err, testError) { t.Fatalf("expected error %v got %v", testError, err) @@ -432,7 +449,7 @@ func TestHandshake(t *testing.T) { }) t.Run("Handle - OK", func(t *testing.T) { - handshakeService, err := handshake.New(signer1, aaddresser, node1Info.BzzAddress.Overlay, networkID, true, nonce, "", true, node1AddrInfo.ID, logger) + handshakeService, err := handshake.New(signer1, aaddresser, node1Info.BzzAddress.Overlay, networkID, true, nonce, nil, "", true, node1AddrInfo.ID, logger) if err != nil { t.Fatal(err) } @@ -461,7 +478,7 @@ func TestHandshake(t *testing.T) { t.Fatal(err) } - res, err := handshakeService.Handle(context.Background(), stream1, node2AddrInfo.Addrs[0], node2AddrInfo.ID) + res, err := handshakeService.Handle(context.Background(), stream1, node2mas) if err != nil { t.Fatal(err) } @@ -490,7 +507,7 @@ func TestHandshake(t *testing.T) { }) t.Run("Handle - read error ", func(t *testing.T) { - handshakeService, err := handshake.New(signer1, aaddresser, node1Info.BzzAddress.Overlay, networkID, true, nil, "", true, node1AddrInfo.ID, logger) + handshakeService, err := handshake.New(signer1, aaddresser, node1Info.BzzAddress.Overlay, networkID, true, nil, nil, "", true, node1AddrInfo.ID, logger) if err != nil { t.Fatal(err) } @@ -498,7 +515,7 @@ func TestHandshake(t *testing.T) { expectedErr := fmt.Errorf("read syn message: %w", testErr) stream := &mock.Stream{} stream.SetReadErr(testErr, 0) - res, err := handshakeService.Handle(context.Background(), stream, node2AddrInfo.Addrs[0], node2AddrInfo.ID) + res, err := handshakeService.Handle(context.Background(), stream, node2mas) if err == nil || err.Error() != expectedErr.Error() { t.Fatal("expected:", expectedErr, "got:", err) } @@ -509,7 +526,7 @@ func TestHandshake(t *testing.T) { }) t.Run("Handle - write error ", func(t *testing.T) { - handshakeService, err := handshake.New(signer1, aaddresser, node1Info.BzzAddress.Overlay, networkID, true, nil, "", true, node1AddrInfo.ID, logger) + handshakeService, err := handshake.New(signer1, aaddresser, node1Info.BzzAddress.Overlay, networkID, true, nil, nil, "", true, node1AddrInfo.ID, logger) if err != nil { t.Fatal(err) } @@ -525,7 +542,7 @@ func TestHandshake(t *testing.T) { t.Fatal(err) } - res, err := handshakeService.Handle(context.Background(), stream, node2AddrInfo.Addrs[0], node2AddrInfo.ID) + res, err := handshakeService.Handle(context.Background(), stream, node2mas) if err == nil || err.Error() != expectedErr.Error() { t.Fatal("expected:", expectedErr, "got:", err) } @@ -536,7 +553,7 @@ func TestHandshake(t *testing.T) { }) t.Run("Handle - ack read error ", func(t *testing.T) { - handshakeService, err := handshake.New(signer1, aaddresser, node1Info.BzzAddress.Overlay, networkID, true, nil, "", true, node1AddrInfo.ID, logger) + handshakeService, err := handshake.New(signer1, aaddresser, node1Info.BzzAddress.Overlay, networkID, true, nil, nil, "", true, node1AddrInfo.ID, logger) if err != nil { t.Fatal(err) } @@ -554,7 +571,7 @@ func TestHandshake(t *testing.T) { t.Fatal(err) } - res, err := handshakeService.Handle(context.Background(), stream1, node2AddrInfo.Addrs[0], node2AddrInfo.ID) + res, err := handshakeService.Handle(context.Background(), stream1, node2mas) if err == nil || err.Error() != expectedErr.Error() { t.Fatal("expected:", expectedErr, "got:", err) } @@ -565,7 +582,7 @@ func TestHandshake(t *testing.T) { }) t.Run("Handle - networkID mismatch ", func(t *testing.T) { - handshakeService, err := handshake.New(signer1, aaddresser, node1Info.BzzAddress.Overlay, networkID, true, nil, "", true, node1AddrInfo.ID, logger) + handshakeService, err := handshake.New(signer1, aaddresser, node1Info.BzzAddress.Overlay, networkID, true, nil, nil, "", true, node1AddrInfo.ID, logger) if err != nil { t.Fatal(err) } @@ -593,7 +610,7 @@ func TestHandshake(t *testing.T) { t.Fatal(err) } - res, err := handshakeService.Handle(context.Background(), stream1, node2AddrInfo.Addrs[0], node2AddrInfo.ID) + res, err := handshakeService.Handle(context.Background(), stream1, node2mas) if res != nil { t.Fatal("res should be nil") } @@ -604,7 +621,7 @@ func TestHandshake(t *testing.T) { }) t.Run("Handle - invalid ack", func(t *testing.T) { - handshakeService, err := handshake.New(signer1, aaddresser, node1Info.BzzAddress.Overlay, networkID, true, nil, "", true, node1AddrInfo.ID, logger) + handshakeService, err := handshake.New(signer1, aaddresser, node1Info.BzzAddress.Overlay, networkID, true, nil, nil, "", true, node1AddrInfo.ID, logger) if err != nil { t.Fatal(err) } @@ -632,14 +649,14 @@ func TestHandshake(t *testing.T) { t.Fatal(err) } - _, err = handshakeService.Handle(context.Background(), stream1, node2AddrInfo.Addrs[0], node2AddrInfo.ID) + _, err = handshakeService.Handle(context.Background(), stream1, node2mas) if !errors.Is(err, handshake.ErrInvalidAck) { t.Fatalf("expected %s, got %v", handshake.ErrInvalidAck, err) } }) t.Run("Handle - advertisable error", func(t *testing.T) { - handshakeService, err := handshake.New(signer1, aaddresser, node1Info.BzzAddress.Overlay, networkID, true, nil, "", true, node1AddrInfo.ID, logger) + handshakeService, err := handshake.New(signer1, aaddresser, node1Info.BzzAddress.Overlay, networkID, true, nil, nil, "", true, node1AddrInfo.ID, logger) if err != nil { t.Fatal(err) } @@ -661,7 +678,7 @@ func TestHandshake(t *testing.T) { t.Fatal(err) } - res, err := handshakeService.Handle(context.Background(), stream1, node2AddrInfo.Addrs[0], node2AddrInfo.ID) + res, err := handshakeService.Handle(context.Background(), stream1, node2mas) if !errors.Is(err, testError) { t.Fatal("expected error") } diff --git a/pkg/p2p/libp2p/internal/reacher/metrics.go b/pkg/p2p/libp2p/internal/reacher/metrics.go deleted file mode 100644 index a6019ba4a4e..00000000000 --- a/pkg/p2p/libp2p/internal/reacher/metrics.go +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2021 The Swarm Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package reacher - -import ( - m "github.com/ethersphere/bee/v2/pkg/metrics" - "github.com/prometheus/client_golang/prometheus" -) - -type metrics struct { - Pings *prometheus.CounterVec - PingTime *prometheus.HistogramVec -} - -func newMetrics() metrics { - subsystem := "reacher" - - return metrics{ - Pings: prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: m.Namespace, - Subsystem: subsystem, - Name: "pings", - Help: "Ping counter.", - }, []string{"status"}), - PingTime: prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: m.Namespace, - Subsystem: subsystem, - Name: "ping_timer", - Help: "Ping timer.", - }, []string{"status"}), - } -} - -func (s *reacher) Metrics() []prometheus.Collector { - return m.PrometheusCollectorsFromFields(s.metrics) -} diff --git a/pkg/p2p/libp2p/internal/reacher/reacher.go b/pkg/p2p/libp2p/internal/reacher/reacher.go index 18b1431229f..aa2ec9cf3e5 100644 --- a/pkg/p2p/libp2p/internal/reacher/reacher.go +++ b/pkg/p2p/libp2p/internal/reacher/reacher.go @@ -11,6 +11,7 @@ import ( "sync" "time" + "github.com/ethersphere/bee/v2/pkg/log" "github.com/ethersphere/bee/v2/pkg/p2p" "github.com/ethersphere/bee/v2/pkg/swarm" ma "github.com/multiformats/go-multiaddr" @@ -38,10 +39,10 @@ type reacher struct { pinger p2p.Pinger notifier p2p.ReachableNotifier - wg sync.WaitGroup - metrics metrics + wg sync.WaitGroup options *Options + logger log.Logger } type Options struct { @@ -50,15 +51,14 @@ type Options struct { RetryAfterDuration time.Duration } -func New(streamer p2p.Pinger, notifier p2p.ReachableNotifier, o *Options) *reacher { - +func New(streamer p2p.Pinger, notifier p2p.ReachableNotifier, o *Options, log log.Logger) *reacher { r := &reacher{ newPeer: make(chan struct{}, 1), quit: make(chan struct{}), pinger: streamer, peers: make(map[string]*peer), notifier: notifier, - metrics: newMetrics(), + logger: log.WithName("reacher").Register(), } if o == nil { @@ -130,27 +130,20 @@ func (r *reacher) manage() { } func (r *reacher) ping(c chan *peer, ctx context.Context) { - defer r.wg.Done() - for p := range c { - - now := time.Now() - - ctxt, cancel := context.WithTimeout(ctx, r.options.PingTimeout) - _, err := r.pinger.Ping(ctxt, p.addr) - cancel() - - // ping was successful - if err == nil { - r.metrics.Pings.WithLabelValues("success").Inc() - r.metrics.PingTime.WithLabelValues("success").Observe(time.Since(now).Seconds()) - r.notifier.Reachable(p.overlay, p2p.ReachabilityStatusPublic) - } else { - r.metrics.Pings.WithLabelValues("failure").Inc() - r.metrics.PingTime.WithLabelValues("failure").Observe(time.Since(now).Seconds()) - r.notifier.Reachable(p.overlay, p2p.ReachabilityStatusPrivate) - } + func() { + ctxt, cancel := context.WithTimeout(ctx, r.options.PingTimeout) + defer cancel() + rtt, err := r.pinger.Ping(ctxt, p.addr) + if err != nil { + r.logger.Debug("ping failed", "peer", p.overlay.String(), "addr", p.addr.String(), "error", err) + r.notifier.Reachable(p.overlay, p2p.ReachabilityStatusPrivate) + } else { + r.logger.Debug("ping succeeded", "peer", p.overlay.String(), "addr", p.addr.String(), "rtt", rtt) + r.notifier.Reachable(p.overlay, p2p.ReachabilityStatusPublic) + } + }() } } diff --git a/pkg/p2p/libp2p/internal/reacher/reacher_test.go b/pkg/p2p/libp2p/internal/reacher/reacher_test.go index 73ba1842b2d..a0e55f2fb35 100644 --- a/pkg/p2p/libp2p/internal/reacher/reacher_test.go +++ b/pkg/p2p/libp2p/internal/reacher/reacher_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/ethersphere/bee/v2/pkg/log" "github.com/ethersphere/bee/v2/pkg/p2p" "github.com/ethersphere/bee/v2/pkg/p2p/libp2p/internal/reacher" "github.com/ethersphere/bee/v2/pkg/swarm" @@ -67,12 +68,13 @@ func TestPingSuccess(t *testing.T) { done := make(chan struct{}) mock := newMock(tc.pingFunc, tc.reachableFunc(done)) - r := reacher.New(mock, mock, &defaultOptions) + r := reacher.New(mock, mock, &defaultOptions, log.Noop) testutil.CleanupCloser(t, r) overlay := swarm.RandAddress(t) + addr, _ := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/7071/p2p/16Uiu2HAmTBuJT9LvNmBiQiNoTsxE5mtNy6YG3paw79m94CRa9sRb") - r.Connected(overlay, nil) + r.Connected(overlay, addr) select { case <-time.After(time.Second * 5): @@ -113,10 +115,11 @@ func TestDisconnected(t *testing.T) { mock := newMock(pingFunc, reachableFunc) - r := reacher.New(mock, mock, &defaultOptions) + r := reacher.New(mock, mock, &defaultOptions, log.Noop) testutil.CleanupCloser(t, r) - r.Connected(swarm.RandAddress(t), nil) + addr, _ := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/7072/p2p/16Uiu2HAmTBuJT9LvNmBiQiNoTsxE5mtNy6YG3paw79m94CRa9sRb") + r.Connected(swarm.RandAddress(t), addr) r.Connected(disconnectedOverlay, disconnectedMa) r.Disconnected(disconnectedOverlay) } diff --git a/pkg/p2p/libp2p/libp2p.go b/pkg/p2p/libp2p/libp2p.go index e1091e0b637..7a537913099 100644 --- a/pkg/p2p/libp2p/libp2p.go +++ b/pkg/p2p/libp2p/libp2p.go @@ -12,16 +12,20 @@ import ( "net" "os" "runtime" + "slices" "strconv" "strings" "sync" "time" + ocprom "contrib.go.opencensus.io/exporter/prometheus" + "github.com/coreos/go-semver/semver" "github.com/ethersphere/bee/v2" "github.com/ethersphere/bee/v2/pkg/addressbook" "github.com/ethersphere/bee/v2/pkg/bzz" beecrypto "github.com/ethersphere/bee/v2/pkg/crypto" "github.com/ethersphere/bee/v2/pkg/log" + m2 "github.com/ethersphere/bee/v2/pkg/metrics" "github.com/ethersphere/bee/v2/pkg/p2p" "github.com/ethersphere/bee/v2/pkg/p2p/libp2p/internal/blocklist" "github.com/ethersphere/bee/v2/pkg/p2p/libp2p/internal/breaker" @@ -48,14 +52,11 @@ import ( libp2pping "github.com/libp2p/go-libp2p/p2p/protocol/ping" "github.com/libp2p/go-libp2p/p2p/transport/tcp" ws "github.com/libp2p/go-libp2p/p2p/transport/websocket" - ma "github.com/multiformats/go-multiaddr" + manet "github.com/multiformats/go-multiaddr/net" "github.com/multiformats/go-multistream" - "go.uber.org/atomic" - - ocprom "contrib.go.opencensus.io/exporter/prometheus" - m2 "github.com/ethersphere/bee/v2/pkg/metrics" "github.com/prometheus/client_golang/prometheus" + "go.uber.org/atomic" ) // loggerName is the tree path name of the logger for this package. @@ -74,6 +75,8 @@ const ( defaultLightNodeLimit = 100 peerUserAgentTimeout = time.Second + peerstoreWaitAddrsTimeout = 10 * time.Second + defaultHeadersRWTimeout = 10 * time.Second IncomingStreamCountLimit = 5_000 @@ -303,7 +306,7 @@ func New(ctx context.Context, signer beecrypto.Signer, networkID uint64, overlay advertisableAddresser = natAddrResolver } - handshakeService, err := handshake.New(signer, advertisableAddresser, overlay, networkID, o.FullNode, o.Nonce, o.WelcomeMessage, o.ValidateOverlay, h.ID(), logger) + handshakeService, err := handshake.New(signer, advertisableAddresser, overlay, networkID, o.FullNode, o.Nonce, newHostAddresser(h), o.WelcomeMessage, o.ValidateOverlay, h.ID(), logger) if err != nil { return nil, fmt.Errorf("handshake service: %w", err) } @@ -409,7 +412,22 @@ func (s *Service) handleIncoming(stream network.Stream) { peerID := stream.Conn().RemotePeer() handshakeStream := newStream(stream, s.metrics) - i, err := s.handshakeService.Handle(s.ctx, handshakeStream, stream.Conn().RemoteMultiaddr(), peerID) + + peerMultiaddrs, err := s.peerMultiaddrs(s.ctx, stream.Conn(), peerID) + if err != nil { + s.logger.Debug("stream handler: handshake: build remote multiaddrs", "peer_id", peerID, "error", err) + s.logger.Error(nil, "stream handler: handshake: build remote multiaddrs", "peer_id", peerID) + _ = handshakeStream.Reset() + _ = s.host.Network().ClosePeer(peerID) + return + } + + i, err := s.handshakeService.Handle( + s.ctx, + handshakeStream, + peerMultiaddrs, + handshake.WithBee260Compatibility(s.bee260BackwardCompatibility(peerID)), + ) if err != nil { s.logger.Debug("stream handler: handshake: handle failed", "peer_id", peerID, "error", err) s.logger.Error(nil, "stream handler: handshake: handle failed", "peer_id", peerID) @@ -536,9 +554,7 @@ func (s *Service) handleIncoming(stream network.Stream) { return } - if s.reacher != nil { - s.reacher.Connected(overlay, i.BzzAddress.Underlay) - } + s.notifyReacherConnected(stream, overlay, peerID) peerUserAgent := appendSpace(s.peerUserAgent(s.ctx, peerID)) s.networkStatus.Store(int32(p2p.NetworkStatusAvailable)) @@ -547,10 +563,30 @@ func (s *Service) handleIncoming(stream network.Stream) { s.logger.Debug("stream handler: successfully connected to peer (inbound)", "address", i.BzzAddress.Overlay, "light", i.LightString(), "user_agent", peerUserAgent) } +func (s *Service) notifyReacherConnected(stream network.Stream, overlay swarm.Address, peerID libp2ppeer.ID) { + if s.reacher == nil { + return + } + + peerAddrs := s.host.Peerstore().Addrs(peerID) + connectionAddr := stream.Conn().RemoteMultiaddr() + bestAddr := bzz.SelectBestAdvertisedAddress(peerAddrs, connectionAddr) + + s.logger.Debug("selected reacher address", "peer_id", peerID, "selected_addr", bestAddr.String(), "connection_addr", connectionAddr.String(), "advertised_count", len(peerAddrs)) + + underlay, err := buildFullMA(bestAddr, peerID) + if err != nil { + s.logger.Error(err, "stream handler: unable to build complete peer multiaddr", "peer", overlay, "multiaddr", bestAddr, "peer_id", peerID) + _ = s.Disconnect(overlay, "unable to build complete peer multiaddr") + return + } + s.reacher.Connected(overlay, underlay) +} + func (s *Service) SetPickyNotifier(n p2p.PickyNotifier) { s.handshakeService.SetPicker(n) s.notifier = n - s.reacher = reacher.New(s, n, nil) + s.reacher = reacher.New(s, n, nil, s.logger) } func (s *Service) AddProtocol(p p2p.ProtocolSpec) (err error) { @@ -707,40 +743,67 @@ func buildUnderlayAddress(addr ma.Multiaddr, peerID libp2ppeer.ID) (ma.Multiaddr return addr.Encapsulate(hostAddr), nil } -func (s *Service) Connect(ctx context.Context, addr ma.Multiaddr) (address *bzz.Address, err error) { +func (s *Service) Connect(ctx context.Context, addrs []ma.Multiaddr) (address *bzz.Address, err error) { loggerV1 := s.logger.V(1).Register() defer func() { err = s.determineCurrentNetworkStatus(err) }() - // Extract the peer ID from the multiaddr. - info, err := libp2ppeer.AddrInfoFromP2pAddr(addr) - if err != nil { - return nil, fmt.Errorf("addr from p2p: %w", err) - } + var info *libp2ppeer.AddrInfo + var peerID libp2ppeer.ID + var connectErr error + + // Try to connect to each underlay address one by one. + // + // TODO: investigate the issue when AddrInfo with multiple underlay + // addresses for the same peer is passed to the host.Host.Connect function + // and reachabiltiy Private is emitted on libp2p EventBus(), which results + // in weaker connectivity and failures in some integration tests. + for _, addr := range addrs { + // Extract the peer ID from the multiaddr. + ai, err := libp2ppeer.AddrInfoFromP2pAddr(addr) + if err != nil { + return nil, fmt.Errorf("addr from p2p: %w", err) + } - hostAddr, err := buildHostAddress(info.ID) - if err != nil { - return nil, fmt.Errorf("build host address: %w", err) - } + info = ai + peerID = ai.ID + + hostAddr, err := buildHostAddress(info.ID) + if err != nil { + return nil, fmt.Errorf("build host address: %w", err) + } - remoteAddr := addr.Decapsulate(hostAddr) + remoteAddr := addr.Decapsulate(hostAddr) - if overlay, found := s.peers.isConnected(info.ID, remoteAddr); found { - address = &bzz.Address{ - Overlay: overlay, - Underlay: addr, + if overlay, found := s.peers.isConnected(info.ID, remoteAddr); found { + address = &bzz.Address{ + Overlay: overlay, + Underlays: []ma.Multiaddr{addr}, + } + return address, p2p.ErrAlreadyConnected } - return address, p2p.ErrAlreadyConnected - } - if err := s.connectionBreaker.Execute(func() error { return s.host.Connect(ctx, *info) }); err != nil { - if errors.Is(err, breaker.ErrClosed) { - s.metrics.ConnectBreakerCount.Inc() - return nil, p2p.NewConnectionBackoffError(err, s.connectionBreaker.ClosedUntil()) + if err := s.connectionBreaker.Execute(func() error { return s.host.Connect(ctx, *info) }); err != nil { + if errors.Is(err, breaker.ErrClosed) { + s.metrics.ConnectBreakerCount.Inc() + return nil, p2p.NewConnectionBackoffError(err, s.connectionBreaker.ClosedUntil()) + } + s.logger.Warning("libp2p connect", "peer_id", peerID, "underlay", info.Addrs, "error", err) + connectErr = err + continue } - return nil, err + + connectErr = nil + } + + if connectErr != nil { + return nil, fmt.Errorf("libp2p connect: %w", err) + } + + if info == nil { + return nil, fmt.Errorf("unable to identify peer from addresses: %v", addrs) } stream, err := s.newStreamForPeerID(ctx, info.ID, handshake.ProtocolName, handshake.ProtocolVersion, handshake.StreamName) @@ -750,7 +813,20 @@ func (s *Service) Connect(ctx context.Context, addr ma.Multiaddr) (address *bzz. } handshakeStream := newStream(stream, s.metrics) - i, err := s.handshakeService.Handshake(ctx, handshakeStream, stream.Conn().RemoteMultiaddr(), stream.Conn().RemotePeer()) + + peerMultiaddrs, err := s.peerMultiaddrs(ctx, stream.Conn(), peerID) + if err != nil { + _ = handshakeStream.Reset() + _ = s.host.Network().ClosePeer(peerID) + return nil, fmt.Errorf("build peer multiaddrs: %w", err) + } + + i, err := s.handshakeService.Handshake( + s.ctx, + handshakeStream, + peerMultiaddrs, + handshake.WithBee260Compatibility(s.bee260BackwardCompatibility(peerID)), + ) if err != nil { _ = handshakeStream.Reset() _ = s.host.Network().ClosePeer(info.ID) @@ -823,14 +899,11 @@ func (s *Service) Connect(ctx context.Context, addr ma.Multiaddr) (address *bzz. s.metrics.CreatedConnectionCount.Inc() - if s.reacher != nil { - s.reacher.Connected(overlay, i.BzzAddress.Underlay) - } - - peerUserAgent := appendSpace(s.peerUserAgent(ctx, info.ID)) + s.notifyReacherConnected(stream, overlay, peerID) - loggerV1.Debug("successfully connected to peer (outbound)", "addresses", i.BzzAddress.ShortString(), "light", i.LightString(), "user_agent", peerUserAgent) - s.logger.Debug("successfully connected to peer (outbound)", "address", i.BzzAddress.Overlay, "light", i.LightString(), "user_agent", peerUserAgent) + peerUA := appendSpace(s.peerUserAgent(ctx, peerID)) + loggerV1.Debug("successfully connected to peer (outbound)", "addresses", i.BzzAddress.ShortString(), "light", i.LightString(), "user_agent", peerUA) + s.logger.Debug("successfully connected to peer (outbound)", "address", overlay, "light", i.LightString(), "user_agent", peerUA) return i.BzzAddress, nil } @@ -1118,6 +1191,45 @@ func (s *Service) determineCurrentNetworkStatus(err error) error { return err } +// peerMultiaddrs builds full multiaddresses for a peer given information from +// libp2p host peerstore and falling back to the remote address from the +// connection. +func (s *Service) peerMultiaddrs(ctx context.Context, conn network.Conn, peerID libp2ppeer.ID) ([]ma.Multiaddr, error) { + waitPeersCtx, cancel := context.WithTimeout(ctx, peerstoreWaitAddrsTimeout) + defer cancel() + + peerMultiaddrs, err := buildFullMAs(waitPeerAddrs(waitPeersCtx, s.host.Peerstore(), peerID), peerID) + if err != nil { + return nil, fmt.Errorf("build peer multiaddrs: %w", err) + } + + if len(peerMultiaddrs) == 0 { + fullRemoteAddress, err := buildFullMA(conn.RemoteMultiaddr(), peerID) + if err != nil { + return nil, fmt.Errorf("build full remote peer multi address: %w", err) + } + peerMultiaddrs = append(peerMultiaddrs, fullRemoteAddress) + } + + return peerMultiaddrs, nil +} + +var version270 = *semver.Must(semver.NewVersion("2.7.0")) + +func (s *Service) bee260BackwardCompatibility(peerID libp2ppeer.ID) bool { + userAgent := s.peerUserAgent(s.ctx, peerID) + p := strings.SplitN(userAgent, " ", 2) + if len(p) != 2 { + return false + } + version := strings.TrimPrefix(p[0], "bee/") + v, err := semver.NewVersion(version) + if err != nil { + return false + } + return v.LessThan(version270) +} + // appendSpace adds a leading space character if the string is not empty. // It is useful for constructing log messages with conditional substrings. func appendSpace(s string) string { @@ -1177,3 +1289,72 @@ func isNetworkOrHostUnreachableError(err error) bool { } return false } + +type hostAddresser struct { + host host.Host +} + +func newHostAddresser(host host.Host) *hostAddresser { + return &hostAddresser{ + host: host, + } +} + +func (h *hostAddresser) AdvertizableAddrs() ([]ma.Multiaddr, error) { + addrs := make([]ma.Multiaddr, 0) + for _, a := range h.host.Addrs() { + if manet.IsIPLoopback(a) { + continue + } + addrs = append(addrs, a) + } + return buildFullMAs(addrs, h.host.ID()) +} + +func buildFullMAs(addrs []ma.Multiaddr, peerID libp2ppeer.ID) ([]ma.Multiaddr, error) { + fullMAs := make([]ma.Multiaddr, 0) + for _, addr := range addrs { + res, err := buildFullMA(addr, peerID) + if err != nil { + return nil, err + } + if slices.ContainsFunc(fullMAs, func(a ma.Multiaddr) bool { + return a.Equal(res) + }) { + continue + } + fullMAs = append(fullMAs, res) + } + return fullMAs, nil +} + +func buildFullMA(addr ma.Multiaddr, peerID libp2ppeer.ID) (ma.Multiaddr, error) { + if _, err := addr.ValueForProtocol(ma.P_P2P); err == nil { + return addr, nil + } + return ma.NewMultiaddr(fmt.Sprintf("%s/p2p/%s", addr.String(), peerID.String())) +} + +// waitPeerAddrs is used to reliably get remote addresses from libp2p peerstore +// as sometimes addresses are not available soon enough from its Addrs() method. +func waitPeerAddrs(ctx context.Context, s peerstore.Peerstore, peerID libp2ppeer.ID) []ma.Multiaddr { + ctx, cancel := context.WithCancel(ctx) // cancel the addrStream when this function exits + defer cancel() + + // ensure that the AddrStream will receive addresses by creating it before Addrs() is called + // this may happen just after the connection is established and peerstore is not updated + addrStream := s.AddrStream(ctx, peerID) + + addrs := s.Addrs(peerID) + if len(addrs) > 0 { + return addrs + } + + select { + case addr := <-addrStream: + // return the first address as it arrives + return []ma.Multiaddr{addr} + case <-ctx.Done(): + return s.Addrs(peerID) + } +} diff --git a/pkg/p2p/libp2p/libp2p_test.go b/pkg/p2p/libp2p/libp2p_test.go index c467247547e..85208272f1c 100644 --- a/pkg/p2p/libp2p/libp2p_test.go +++ b/pkg/p2p/libp2p/libp2p_test.go @@ -152,12 +152,12 @@ func expectPeersEventually(t *testing.T, s *libp2p.Service, addrs ...swarm.Addre } } -func serviceUnderlayAddress(t *testing.T, s *libp2p.Service) multiaddr.Multiaddr { +func serviceUnderlayAddress(t *testing.T, s *libp2p.Service) []multiaddr.Multiaddr { t.Helper() addrs, err := s.Addresses() if err != nil { t.Fatal(err) } - return addrs[0] + return addrs } diff --git a/pkg/p2p/libp2p/protocols_test.go b/pkg/p2p/libp2p/protocols_test.go index 50a9d9c2970..30c9f8504f1 100644 --- a/pkg/p2p/libp2p/protocols_test.go +++ b/pkg/p2p/libp2p/protocols_test.go @@ -421,7 +421,7 @@ func TestPing(t *testing.T) { addr := serviceUnderlayAddress(t, s1) - if _, err := s2.Ping(ctx, addr); err != nil { + if _, err := s2.Ping(ctx, addr[0]); err != nil { t.Fatal(err) } } diff --git a/pkg/p2p/mock/mock.go b/pkg/p2p/mock/mock.go index b5fc33f5373..59197b30ba0 100644 --- a/pkg/p2p/mock/mock.go +++ b/pkg/p2p/mock/mock.go @@ -18,7 +18,7 @@ import ( // Service is the mock of a P2P Service type Service struct { addProtocolFunc func(p2p.ProtocolSpec) error - connectFunc func(ctx context.Context, addr ma.Multiaddr) (address *bzz.Address, err error) + connectFunc func(ctx context.Context, addr []ma.Multiaddr) (address *bzz.Address, err error) disconnectFunc func(overlay swarm.Address, reason string) error peersFunc func() []p2p.Peer blocklistedPeersFunc func() ([]p2p.BlockListedPeer, error) @@ -38,7 +38,7 @@ func WithAddProtocolFunc(f func(p2p.ProtocolSpec) error) Option { } // WithConnectFunc sets the mock implementation of the Connect function -func WithConnectFunc(f func(ctx context.Context, addr ma.Multiaddr) (address *bzz.Address, err error)) Option { +func WithConnectFunc(f func(ctx context.Context, addr []ma.Multiaddr) (address *bzz.Address, err error)) Option { return optionFunc(func(s *Service) { s.connectFunc = f }) @@ -108,7 +108,7 @@ func (s *Service) AddProtocol(spec p2p.ProtocolSpec) error { return s.addProtocolFunc(spec) } -func (s *Service) Connect(ctx context.Context, addr ma.Multiaddr) (address *bzz.Address, err error) { +func (s *Service) Connect(ctx context.Context, addr []ma.Multiaddr) (address *bzz.Address, err error) { if s.connectFunc == nil { return nil, errors.New("function Connect not configured") } diff --git a/pkg/p2p/p2p.go b/pkg/p2p/p2p.go index a85b3d9d04b..b709ffa5113 100644 --- a/pkg/p2p/p2p.go +++ b/pkg/p2p/p2p.go @@ -61,7 +61,7 @@ var ErrNetworkUnavailable = errors.New("network unavailable") type Service interface { AddProtocol(ProtocolSpec) error // Connect to a peer but do not notify topology about the established connection. - Connect(ctx context.Context, addr ma.Multiaddr) (address *bzz.Address, err error) + Connect(ctx context.Context, addrs []ma.Multiaddr) (address *bzz.Address, err error) Disconnecter Peers() []Peer Blocklisted(swarm.Address) (bool, error) diff --git a/pkg/topology/kademlia/kademlia.go b/pkg/topology/kademlia/kademlia.go index e045c9658fe..4761b15519a 100644 --- a/pkg/topology/kademlia/kademlia.go +++ b/pkg/topology/kademlia/kademlia.go @@ -36,9 +36,9 @@ import ( const loggerName = "kademlia" const ( - maxConnAttempts = 1 // when there is maxConnAttempts failed connect calls for a given peer it is considered non-connectable - maxBootNodeAttempts = 3 // how many attempts to dial to boot-nodes before giving up - maxNeighborAttempts = 3 // how many attempts to dial to boot-nodes before giving up + maxConnAttempts = 4 // when there is maxConnAttempts failed connect calls for a given peer it is considered non-connectable + maxBootNodeAttempts = 6 // how many attempts to dial to boot-nodes before giving up + maxNeighborAttempts = 6 // how many attempts to dial to boot-nodes before giving up addPeerBatchSize = 500 @@ -55,7 +55,7 @@ const ( defaultSaturationPeers = 8 defaultOverSaturationPeers = 18 defaultBootNodeOverSaturationPeers = 20 - defaultShortRetry = 30 * time.Second + defaultShortRetry = 10 * time.Second defaultTimeToRetry = 2 * defaultShortRetry defaultPruneWakeup = 5 * time.Minute defaultBroadcastBinSize = 2 @@ -429,16 +429,16 @@ func (k *Kad) connectionAttemptsHandler(ctx context.Context, wg *sync.WaitGroup, } } - switch err = k.connect(ctx, peer.addr, bzzAddr.Underlay); { + switch err = k.connect(ctx, peer.addr, bzzAddr.Underlays); { case errors.Is(err, p2p.ErrNetworkUnavailable): - k.logger.Debug("network unavailable when reaching peer", "peer_overlay_address", peer.addr, "peer_underlay_address", bzzAddr.Underlay) + k.logger.Debug("network unavailable when reaching peer", "peer_overlay_address", peer.addr, "peer_underlay_addresses", bzzAddr.Underlays) return case errors.Is(err, errPruneEntry): - k.logger.Debug("dial to light node", "peer_overlay_address", peer.addr, "peer_underlay_address", bzzAddr.Underlay) + k.logger.Debug("dial to light node", "peer_overlay_address", peer.addr, "peer_underlay_addresses", bzzAddr.Underlays) remove(peer) return case errors.Is(err, errOverlayMismatch): - k.logger.Debug("overlay mismatch has occurred", "peer_overlay_address", peer.addr, "peer_underlay_address", bzzAddr.Underlay) + k.logger.Debug("overlay mismatch has occurred", "peer_overlay_address", peer.addr, "peer_underlay_addresses", bzzAddr.Underlays) remove(peer) return case errors.Is(err, p2p.ErrPeerBlocklisted): @@ -820,7 +820,7 @@ func (k *Kad) connectBootNodes(ctx context.Context) { if attempts >= maxBootNodeAttempts { return true, nil } - bzzAddress, err := k.p2p.Connect(ctx, addr) + bzzAddress, err := k.p2p.Connect(ctx, []ma.Multiaddr{addr}) attempts++ k.metrics.TotalBootNodesConnectionAttempts.Inc() @@ -958,7 +958,7 @@ func (k *Kad) recalcDepth() { // connect connects to a peer and gossips its address to our connected peers, // as well as sends the peers we are connected to the newly connected peer -func (k *Kad) connect(ctx context.Context, peer swarm.Address, ma ma.Multiaddr) error { +func (k *Kad) connect(ctx context.Context, peer swarm.Address, ma []ma.Multiaddr) error { k.logger.Debug("attempting connect to peer", "peer_address", peer) ctx, cancel := context.WithTimeout(ctx, peerConnectionAttemptTimeout) @@ -983,7 +983,7 @@ func (k *Kad) connect(ctx context.Context, peer swarm.Address, ma ma.Multiaddr) case errors.Is(err, p2p.ErrPeerBlocklisted): return err case err != nil: - k.logger.Debug("could not connect to peer", "peer_address", peer, "error", err) + k.logger.Info("could not connect to peer", "peer_address", peer, "error", err) retryTime := time.Now().Add(k.opt.TimeToRetry) var e *p2p.ConnectionBackoffError diff --git a/pkg/topology/kademlia/kademlia_test.go b/pkg/topology/kademlia/kademlia_test.go index 441ac24ec49..4d534ddade8 100644 --- a/pkg/topology/kademlia/kademlia_test.go +++ b/pkg/topology/kademlia/kademlia_test.go @@ -11,6 +11,7 @@ import ( "math" "math/rand" "reflect" + "strconv" "sync" "sync/atomic" "testing" @@ -825,7 +826,7 @@ func TestAddressBookPrune(t *testing.T) { } testutil.CleanupCloser(t, kad) - nonConnPeer, err := bzz.NewAddress(signer, nonConnectableAddress, swarm.RandAddressAt(t, base, 1), 0, nil) + nonConnPeer, err := bzz.NewAddress(signer, []ma.Multiaddr{nonConnectableAddress}, swarm.RandAddressAt(t, base, 1), 0, nil) if err != nil { t.Fatal(err) } @@ -836,11 +837,12 @@ func TestAddressBookPrune(t *testing.T) { // add non connectable peer, check connection and failed connection counters kad.AddPeers(nonConnPeer.Overlay) - kad.Trigger() - kad.Trigger() + for range 5 { + kad.Trigger() + } waitCounter(t, &conns, 0) - waitCounter(t, &failedConns, 3) + waitCounter(t, &failedConns, 6) _, err = ab.Get(nonConnPeer.Overlay) if !errors.Is(err, addressbook.ErrNotFound) { @@ -903,7 +905,7 @@ func TestAddressBookQuickPrune_FLAKY(t *testing.T) { time.Sleep(100 * time.Millisecond) - nonConnPeer, err := bzz.NewAddress(signer, nonConnectableAddress, swarm.RandAddressAt(t, base, 1), 0, nil) + nonConnPeer, err := bzz.NewAddress(signer, []ma.Multiaddr{nonConnectableAddress}, swarm.RandAddressAt(t, base, 1), 0, nil) if err != nil { t.Fatal(err) } @@ -922,6 +924,13 @@ func TestAddressBookQuickPrune_FLAKY(t *testing.T) { waitCounter(t, &conns, 0) waitCounter(t, &failedConns, 1) + // we need to trigger connection attempts maxConnAttempts times + for range 3 { + time.Sleep(10 * time.Millisecond) + kad.Trigger() + waitCounter(t, &failedConns, 1) + } + _, err = ab.Get(nonConnPeer.Overlay) if !errors.Is(err, addressbook.ErrNotFound) { t.Fatal(err) @@ -1237,7 +1246,7 @@ func TestStart(t *testing.T) { if err != nil { t.Fatal(err) } - bzzAddr, err := bzz.NewAddress(signer, multiaddr, peer, 0, nil) + bzzAddr, err := bzz.NewAddress(signer, []ma.Multiaddr{multiaddr}, peer, 0, nil) if err != nil { t.Fatal(err) } @@ -2026,11 +2035,14 @@ func p2pMock(t *testing.T, ab addressbook.Interface, signer beeCrypto.Signer, co t.Helper() p2ps := p2pmock.New( - p2pmock.WithConnectFunc(func(ctx context.Context, addr ma.Multiaddr) (*bzz.Address, error) { - if addr.Equal(nonConnectableAddress) { - _ = atomic.AddInt32(failedCounter, 1) - return nil, errors.New("non reachable node") + p2pmock.WithConnectFunc(func(ctx context.Context, addrs []ma.Multiaddr) (*bzz.Address, error) { + for _, addr := range addrs { + if addr.Equal(nonConnectableAddress) { + _ = atomic.AddInt32(failedCounter, 1) + return nil, errors.New("non reachable node") + } } + if counter != nil { _ = atomic.AddInt32(counter, 1) } @@ -2041,13 +2053,13 @@ func p2pMock(t *testing.T, ab addressbook.Interface, signer beeCrypto.Signer, co } for _, a := range addresses { - if a.Underlay.Equal(addr) { + if bzz.AreUnderlaysEqual(a.Underlays, addrs) { return &a, nil } } address := swarm.RandAddress(t) - bzzAddr, err := bzz.NewAddress(signer, addr, address, 0, nil) + bzzAddr, err := bzz.NewAddress(signer, addrs, address, 0, nil) if err != nil { return nil, err } @@ -2077,12 +2089,8 @@ const underlayBase = "/ip4/127.0.0.1/tcp/1634/dns/" func connectOne(t *testing.T, signer beeCrypto.Signer, k *kademlia.Kad, ab addressbook.Putter, peer swarm.Address, expErr error) { t.Helper() - multiaddr, err := ma.NewMultiaddr(underlayBase + peer.String()) - if err != nil { - t.Fatal(err) - } - - bzzAddr, err := bzz.NewAddress(signer, multiaddr, peer, 0, nil) + underlays := generateMultipleUnderlays(t, 3, underlayBase+peer.String()) + bzzAddr, err := bzz.NewAddress(signer, underlays, peer, 0, nil) if err != nil { t.Fatal(err) } @@ -2102,7 +2110,7 @@ func addOne(t *testing.T, signer beeCrypto.Signer, k *kademlia.Kad, ab addressbo if err != nil { t.Fatal(err) } - bzzAddr, err := bzz.NewAddress(signer, multiaddr, peer, 0, nil) + bzzAddr, err := bzz.NewAddress(signer, []ma.Multiaddr{multiaddr}, peer, 0, nil) if err != nil { t.Fatal(err) } @@ -2231,3 +2239,17 @@ func ptrInt(v int) *int { func ptrDuration(v time.Duration) *time.Duration { return &v } + +func generateMultipleUnderlays(t *testing.T, n int, baseUnderlay string) []ma.Multiaddr { + t.Helper() + underlays := make([]ma.Multiaddr, n) + + for i := 0; i < n; i++ { + multiaddr, err := ma.NewMultiaddr(baseUnderlay + strconv.Itoa(i)) + if err != nil { + t.Fatal(err) + } + underlays[i] = multiaddr + } + return underlays +} From 9c7116d538e20ea3693b0dca6ffb8adb7095710d Mon Sep 17 00:00:00 2001 From: Akrem Chabchoub <121046693+akrem-chabchoub@users.noreply.github.com> Date: Mon, 3 Nov 2025 15:14:18 +0100 Subject: [PATCH 44/96] test(storage-incentives): add synctest to TestAgent (#5250) --- pkg/storageincentives/agent_test.go | 101 ++++++++++++++++------------ 1 file changed, 57 insertions(+), 44 deletions(-) diff --git a/pkg/storageincentives/agent_test.go b/pkg/storageincentives/agent_test.go index 88db0e3b3d4..6449ede9059 100644 --- a/pkg/storageincentives/agent_test.go +++ b/pkg/storageincentives/agent_test.go @@ -10,6 +10,7 @@ import ( "math/big" "sync" "testing" + "testing/synctest" "time" "github.com/ethereum/go-ethereum/common" @@ -95,66 +96,66 @@ func TestAgent(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - t.Parallel() + synctest.Test(t, func(t *testing.T) { + wait := make(chan struct{}, 1) + addr := swarm.RandAddress(t) + + backend := &mockchainBackend{ + limit: tc.limit, + limitCallback: func() { + wait <- struct{}{} + }, + incrementBy: tc.incrementBy, + block: tc.blocksPerRound, + balance: tc.balance, + } - wait := make(chan struct{}) - addr := swarm.RandAddress(t) + var radius uint8 = 8 - backend := &mockchainBackend{ - limit: tc.limit, - limitCallback: func() { - select { - case wait <- struct{}{}: - default: - } - }, - incrementBy: tc.incrementBy, - block: tc.blocksPerRound, - balance: tc.balance, - } + contract := &mockContract{t: t, expectedRadius: radius + tc.doubling} - var radius uint8 = 8 + service, _ := createService(t, addr, backend, contract, tc.blocksPerRound, tc.blocksPerPhase, radius, tc.doubling) + testutil.CleanupCloser(t, service) - contract := &mockContract{t: t, expectedRadius: radius + tc.doubling} + <-wait - service, _ := createService(t, addr, backend, contract, tc.blocksPerRound, tc.blocksPerPhase, radius, tc.doubling) - testutil.CleanupCloser(t, service) + synctest.Wait() - <-wait + calls := contract.getCalls() - if !tc.expectedCalls { - if len(contract.callsList) > 0 { - t.Fatal("got unexpected calls") - } else { + if !tc.expectedCalls { + if len(calls) > 0 { + t.Fatal("got unexpected calls") + } return } - } - assertOrder := func(t *testing.T, want, got contractCall) { - t.Helper() - if want != got { - t.Fatalf("expected call %s, got %s", want, got) + if len(calls) == 0 { + t.Fatal("expected calls but got none") } - } - contract.mtx.Lock() - defer contract.mtx.Unlock() + assertOrder := func(t *testing.T, want, got contractCall) { + t.Helper() + if want != got { + t.Fatalf("expected call %s, got %s", want, got) + } + } - prevCall := contract.callsList[0] + prevCall := calls[0] - for i := 1; i < len(contract.callsList); i++ { + for i := 1; i < len(calls); i++ { + switch calls[i] { + case isWinnerCall: + assertOrder(t, revealCall, prevCall) + case revealCall: + assertOrder(t, commitCall, prevCall) + case commitCall: + assertOrder(t, isWinnerCall, prevCall) + } - switch contract.callsList[i] { - case isWinnerCall: - assertOrder(t, revealCall, prevCall) - case revealCall: - assertOrder(t, commitCall, prevCall) - case commitCall: - assertOrder(t, isWinnerCall, prevCall) + prevCall = calls[i] } - - prevCall = contract.callsList[i] - } + }) }) } } @@ -276,6 +277,18 @@ type mockContract struct { t *testing.T } +// getCalls returns a snapshot of the calls list +// even after synctest.Wait() all goroutines are blocked, we still should use locking +// for defensive programming. +func (m *mockContract) getCalls() []contractCall { + m.mtx.Lock() + defer m.mtx.Unlock() + // return a copy to avoid external modifications + calls := make([]contractCall, len(m.callsList)) + copy(calls, m.callsList) + return calls +} + func (m *mockContract) ReserveSalt(context.Context) ([]byte, error) { return nil, nil } From 02704a1a9a05084c65ba6d5741ddcfcbeda4dd66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ljubi=C5=A1a=20Ga=C4=8Devi=C4=87?= <35105035+gacevicljubisa@users.noreply.github.com> Date: Tue, 4 Nov 2025 15:05:47 +0100 Subject: [PATCH 45/96] revert(pushsync): cap overdraft backoff delay with jitter (#5275) --- pkg/pushsync/backoff_test.go | 82 ------------------------------------ pkg/pushsync/export_test.go | 11 ++--- pkg/pushsync/pushsync.go | 58 +++++-------------------- 3 files changed, 14 insertions(+), 137 deletions(-) delete mode 100644 pkg/pushsync/backoff_test.go diff --git a/pkg/pushsync/backoff_test.go b/pkg/pushsync/backoff_test.go deleted file mode 100644 index 67b1fa668d4..00000000000 --- a/pkg/pushsync/backoff_test.go +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright 2025 The Swarm Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package pushsync_test - -import ( - "testing" - "time" - - "github.com/ethersphere/bee/v2/pkg/pushsync" -) - -func TestCalculateOverdraftBackoff_InitialBounds(t *testing.T) { - const samples = 200 - - base := time.Duration(float64(pushsync.OverDraftRefresh) * pushsync.OverDraftBackoffMultiplier) - lower := time.Duration(float64(base) * (1 - pushsync.OverDraftJitterPercent)) - upper := time.Duration(float64(base) * (1 + pushsync.OverDraftJitterPercent)) - - for range samples { - got := pushsync.CalculateOverdraftBackoff(0) - if got < lower { - t.Fatalf("backoff below lower bound: got=%v lower=%v", got, lower) - } - if got > upper { - t.Fatalf("backoff above upper bound: got=%v upper=%v", got, upper) - } - if got < pushsync.OverDraftRefresh { - t.Fatalf("backoff below minimum refresh: got=%v min=%v", got, pushsync.OverDraftRefresh) - } - } -} - -func TestCalculateOverdraftBackoff_MaxEdgeJittersAroundCap(t *testing.T) { - const samples = 400 - - current := pushsync.MaxOverDraftRefresh * 10 - base := pushsync.MaxOverDraftRefresh - lower := time.Duration(float64(base) * (1 - pushsync.OverDraftJitterPercent)) - upper := time.Duration(float64(base) * (1 + pushsync.OverDraftJitterPercent)) - - sawAboveMax := false - sawBelowMax := false - - for range samples { - got := pushsync.CalculateOverdraftBackoff(current) - if got < lower { - t.Fatalf("backoff below expected jitter band: got=%v lower=%v", got, lower) - } - if got > upper { - t.Fatalf("backoff above expected jitter band: got=%v upper=%v", got, upper) - } - if got > base { - sawAboveMax = true - } - if got < base { - sawBelowMax = true - } - } - - if !sawAboveMax { - t.Fatalf("expected at least one sample above maxOverDraftRefresh; base=%v", base) - } - if !sawBelowMax { - t.Fatalf("expected at least one sample below maxOverDraftRefresh; base=%v", base) - } - t.Logf("seen above max: %v, seen below max: %v", sawAboveMax, sawBelowMax) -} - -func TestCalculateOverdraftBackoff_NeverBelowMin(t *testing.T) { - const samples = 200 - for i := range samples { - got := pushsync.CalculateOverdraftBackoff(0) - if got < pushsync.OverDraftRefresh { - t.Fatalf("backoff below minimum refresh: got=%v min=%v", got, pushsync.OverDraftRefresh) - } - if i < 5 { - t.Logf("min-check sample %d: delay=%v (min %v)", i, got, pushsync.OverDraftRefresh) - } - } -} diff --git a/pkg/pushsync/export_test.go b/pkg/pushsync/export_test.go index 778b76ea97f..840ce5b7849 100644 --- a/pkg/pushsync/export_test.go +++ b/pkg/pushsync/export_test.go @@ -5,12 +5,7 @@ package pushsync var ( - ProtocolName = protocolName - ProtocolVersion = protocolVersion - StreamName = streamName - OverDraftRefresh = overDraftRefresh - MaxOverDraftRefresh = maxOverDraftRefresh - OverDraftBackoffMultiplier = overDraftBackoffMultiplier - OverDraftJitterPercent = overDraftJitterPercent - CalculateOverdraftBackoff = calculateOverdraftBackoff + ProtocolName = protocolName + ProtocolVersion = protocolVersion + StreamName = streamName ) diff --git a/pkg/pushsync/pushsync.go b/pkg/pushsync/pushsync.go index ba10a5c4403..9a9754a9ab3 100644 --- a/pkg/pushsync/pushsync.go +++ b/pkg/pushsync/pushsync.go @@ -10,7 +10,6 @@ import ( "context" "errors" "fmt" - "math/rand/v2" "strconv" "time" @@ -45,13 +44,10 @@ const ( ) const ( - defaultTTL = 30 * time.Second // request time to live - preemptiveInterval = 5 * time.Second // P90 request time to live - skiplistDur = 5 * time.Minute - overDraftRefresh = time.Millisecond * 600 // overdraft refresh duration and initial backoff delay - maxOverDraftRefresh = time.Second * 5 // maximum backoff delay - overDraftBackoffMultiplier = 1.2 // backoff multiplier - overDraftJitterPercent = 0.1 // 10% jitter + defaultTTL = 30 * time.Second // request time to live + preemptiveInterval = 5 * time.Second // P90 request time to live + skiplistDur = 5 * time.Minute + overDraftRefresh = time.Millisecond * 600 ) const ( @@ -319,6 +315,7 @@ func (ps *PushSync) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream) default: ps.metrics.Forwarder.Inc() return fmt.Errorf("handler: push to closest chunk %s: %w", chunkAddress, err) + } } @@ -358,12 +355,10 @@ func (ps *PushSync) pushToClosest(ctx context.Context, ch swarm.Chunk, origin bo ps.metrics.TotalRequests.Inc() var ( - sentErrorsLeft = 1 - preemptiveTicker <-chan time.Time - inflight int - parallelForwards = maxMultiplexForwards - overdraftBackoffDelay time.Duration - overdraftRetryCount int + sentErrorsLeft = 1 + preemptiveTicker <-chan time.Time + inflight int + parallelForwards = maxMultiplexForwards ) if origin { @@ -429,19 +424,10 @@ func (ps *PushSync) pushToClosest(ctx context.Context, ch swarm.Chunk, origin bo continue // there is still an inflight request, wait for it's result } - overdraftBackoffDelay = calculateOverdraftBackoff(overdraftBackoffDelay) - overdraftRetryCount++ - - // log first 3 attempts and then every 5th - if overdraftRetryCount%5 == 1 || overdraftRetryCount <= 3 { - ps.logger.Debug("sleeping to refresh overdraft balance", - "chunk_address", ch.Address(), - "backoff_delay", overdraftBackoffDelay, - "retry_count", overdraftRetryCount) - } + ps.logger.Debug("sleeping to refresh overdraft balance", "chunk_address", ch.Address()) select { - case <-time.After(overdraftBackoffDelay): + case <-time.After(overDraftRefresh): retry() continue case <-ctx.Done(): @@ -458,12 +444,6 @@ func (ps *PushSync) pushToClosest(ctx context.Context, ch swarm.Chunk, origin bo continue } - if overdraftRetryCount > 0 { - ps.logger.Debug("overdraft resolved, resetting backoff", "chunk_address", ch.Address(), "total_retries", overdraftRetryCount) - overdraftBackoffDelay = 0 - overdraftRetryCount = 0 - } - // since we can reach into the neighborhood of the chunk // act as the multiplexer and push the chunk in parallel to multiple peers if swarm.Proximity(peer.Bytes(), ch.Address().Bytes()) >= rad { @@ -698,22 +678,6 @@ func (ps *PushSync) validStampWrapper(f postage.ValidStampFn) postage.ValidStamp } } -// calculateOverdraftBackoff calculates the next backoff delay using exponential backoff with jitter. -// It takes the current backoff delay and returns the next delay, capped at maxOverDraftRefresh. -func calculateOverdraftBackoff(currentDelay time.Duration) time.Duration { - if currentDelay == 0 { - currentDelay = overDraftRefresh - } - - nextDelay := min(time.Duration(float64(currentDelay)*overDraftBackoffMultiplier), maxOverDraftRefresh) - - jitterRange := float64(nextDelay) * overDraftJitterPercent - jitter := (rand.Float64() - 0.5) * 2 * jitterRange - finalDelay := max(time.Duration(float64(nextDelay)+jitter), overDraftRefresh) - - return finalDelay -} - func (s *PushSync) Close() error { return s.errSkip.Close() } From 57a0355e5ff66e3ebc2dd27c18be5c902124cc45 Mon Sep 17 00:00:00 2001 From: Akrem Chabchoub <121046693+akrem-chabchoub@users.noreply.github.com> Date: Wed, 19 Nov 2025 14:15:52 +0100 Subject: [PATCH 46/96] docs(bee-dev): update dev command with deprecation warning and doc link (#5283) --- cmd/bee/cmd/start_dev.go | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/cmd/bee/cmd/start_dev.go b/cmd/bee/cmd/start_dev.go index 4440b3fe441..9c82d11be60 100644 --- a/cmd/bee/cmd/start_dev.go +++ b/cmd/bee/cmd/start_dev.go @@ -14,13 +14,15 @@ import ( "github.com/ethersphere/bee/v2/pkg/node" "github.com/kardianos/service" "github.com/spf13/cobra" + "golang.org/x/term" ) func (c *command) initStartDevCmd() (err error) { cmd := &cobra.Command{ Use: "dev", - Short: "Start a Swarm node in development mode", + Short: "Start in dev mode. WARNING: This command will be deprecated soon.", + Long: "WARNING: This command will be deprecated soon. For more information, please refer to the official documentation: https://docs.ethswarm.org/docs/develop/tools-and-features/bee-dev-mode and check back regularly for updates.", PersistentPreRunE: c.CheckUnknownParams, RunE: func(cmd *cobra.Command, args []string) (err error) { if len(args) > 0 { @@ -54,6 +56,26 @@ func (c *command) initStartDevCmd() (err error) { fmt.Println(beeASCII) fmt.Println() + + warningBox := ` +╔══════════════════════════════════════════════════════════════════════════════╗ +║ ║ +║ WARNING: This command will be deprecated soon. ║ +║ ║ +║ For more information, please refer to the official documentation: ║ +║ https://docs.ethswarm.org/docs/develop/tools-and-features/bee-dev-mode ║ +║ ║ +║ Please check back regularly for updates. ║ +║ ║ +╚══════════════════════════════════════════════════════════════════════════════╝ +` + if term.IsTerminal(int(os.Stdout.Fd())) { + fmt.Print("\u001b[33m" + warningBox + "\u001b[0m") + } else { + fmt.Print(warningBox) + } + fmt.Println() + fmt.Println("Starting in development mode") fmt.Println() From 48ded1a6f57b8af3ceeb38503d26f24c536f0ecd Mon Sep 17 00:00:00 2001 From: Akrem Chabchoub <121046693+akrem-chabchoub@users.noreply.github.com> Date: Thu, 20 Nov 2025 16:27:23 +0100 Subject: [PATCH 47/96] feat(chequebook): add funding URL for mainnet users (#5284) Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- pkg/settlement/swap/chequebook/init.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/settlement/swap/chequebook/init.go b/pkg/settlement/swap/chequebook/init.go index 8903a17c8bf..8c6b1563fae 100644 --- a/pkg/settlement/swap/chequebook/init.go +++ b/pkg/settlement/swap/chequebook/init.go @@ -98,9 +98,16 @@ func checkBalance( msg := fmt.Sprintf("cannot continue until there is at least min %s available on address", swarmTokenName) logger.Warning(msg, "min_amount", neededERC20, "address", overlayEthAddress) } + if chainId == chaincfg.Testnet.ChainID { logger.Warning("learn how to fund your node by visiting our docs at https://docs.ethswarm.org/docs/installation/fund-your-node") } + + if chainId == chaincfg.Mainnet.ChainID { + fundingURL := fmt.Sprintf("https://fund.ethswarm.org/?destination=%s&intent=initial-funding", overlayEthAddress.Hex()) + logger.Info(fmt.Sprintf("fund your node using the funding URL: %s", fundingURL)) + } + select { case <-time.After(balanceCheckBackoffDuration): case <-timeoutCtx.Done(): From 54b963adb054bbb0e4bec9c20b6eb45e490d3b41 Mon Sep 17 00:00:00 2001 From: Akrem Chabchoub <121046693+akrem-chabchoub@users.noreply.github.com> Date: Sun, 23 Nov 2025 13:01:26 +0100 Subject: [PATCH 48/96] test(spinlock): add synctest to wait test (#5260) --- pkg/spinlock/wait_test.go | 40 +++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/pkg/spinlock/wait_test.go b/pkg/spinlock/wait_test.go index 090543723b1..3091da15ddb 100644 --- a/pkg/spinlock/wait_test.go +++ b/pkg/spinlock/wait_test.go @@ -7,37 +7,37 @@ package spinlock_test import ( "errors" "testing" + "testing/synctest" "time" "github.com/ethersphere/bee/v2/pkg/spinlock" ) func TestWait(t *testing.T) { - t.Parallel() t.Run("timed out", func(t *testing.T) { - t.Parallel() - - err := spinlock.Wait(time.Millisecond*20, func() bool { return false }) - if !errors.Is(err, spinlock.ErrTimedOut) { - t.Fatal("expecting to time out") - } + synctest.Test(t, func(t *testing.T) { + err := spinlock.Wait(time.Millisecond*20, func() bool { return false }) + if !errors.Is(err, spinlock.ErrTimedOut) { + t.Fatal("expecting to time out") + } + }) }) t.Run("condition satisfied", func(t *testing.T) { - t.Parallel() - - spinStartTime := time.Now() - condCallCount := 0 - err := spinlock.Wait(time.Millisecond*200, func() bool { - condCallCount++ - return time.Since(spinStartTime) >= time.Millisecond*100 + synctest.Test(t, func(t *testing.T) { + spinStartTime := time.Now() + condCallCount := 0 + err := spinlock.Wait(time.Millisecond*200, func() bool { + condCallCount++ + return time.Since(spinStartTime) >= time.Millisecond*100 + }) + if err != nil { + t.Fatal("expecting to end wait without time out") + } + if condCallCount == 0 { + t.Fatal("expecting condition function to be called") + } }) - if err != nil { - t.Fatal("expecting to end wait without time out") - } - if condCallCount == 0 { - t.Fatal("expecting condition function to be called") - } }) } From 72ed53ff750bf73305cbc3a14a3d97b4e2f72b67 Mon Sep 17 00:00:00 2001 From: Akrem Chabchoub <121046693+akrem-chabchoub@users.noreply.github.com> Date: Sun, 23 Nov 2025 22:35:50 +0100 Subject: [PATCH 49/96] test(blocker): add synctest to blocker pkg (#5258) --- pkg/blocker/blocker_test.go | 133 ++++++++++++++++++------------------ 1 file changed, 67 insertions(+), 66 deletions(-) diff --git a/pkg/blocker/blocker_test.go b/pkg/blocker/blocker_test.go index d31a815322d..a938204de2d 100644 --- a/pkg/blocker/blocker_test.go +++ b/pkg/blocker/blocker_test.go @@ -7,6 +7,7 @@ package blocker_test import ( "os" "testing" + "testing/synctest" "time" "go.uber.org/goleak" @@ -36,98 +37,98 @@ func TestMain(m *testing.M) { } func TestBlocksAfterFlagTimeout(t *testing.T) { - t.Parallel() + synctest.Test(t, func(t *testing.T) { + addr := swarm.RandAddress(t) + blockedC := make(chan swarm.Address, 10) - addr := swarm.RandAddress(t) - blockedC := make(chan swarm.Address, 10) + mock := mockBlockLister(func(a swarm.Address, d time.Duration, r string) error { + blockedC <- a - mock := mockBlockLister(func(a swarm.Address, d time.Duration, r string) error { - blockedC <- a + if d != blockTime { + t.Fatalf("block time: want %v, got %v", blockTime, d) + } - if d != blockTime { - t.Fatalf("block time: want %v, got %v", blockTime, d) - } - - return nil - }) + return nil + }) - b := blocker.New(mock, flagTime, blockTime, time.Millisecond, nil, log.Noop) - testutil.CleanupCloser(t, b) + b := blocker.New(mock, flagTime, blockTime, time.Millisecond, nil, log.Noop) + testutil.CleanupCloser(t, b) - // Flagging address shouldn't block it immediately - b.Flag(addr) - if len(blockedC) != 0 { - t.Fatal("blocker did not wait flag duration") - } + // Flagging address shouldn't block it immediately + b.Flag(addr) + if len(blockedC) != 0 { + t.Fatal("blocker did not wait flag duration") + } - time.Sleep(flagTime / 2) - b.Flag(addr) // check that this flag call does not override previous call - if len(blockedC) != 0 { - t.Fatal("blocker did not wait flag duration") - } + synctest.Wait() + b.Flag(addr) // check that this flag call does not override previous call + if len(blockedC) != 0 { + t.Fatal("blocker did not wait flag duration") + } - // Suspending current goroutine and expect that in this interval - // block listener was called to block flagged address - time.Sleep(flagTime * 3) + // Suspending current goroutine and expect that in this interval + // block listener was called to block flagged address + synctest.Wait() - if a := <-blockedC; !a.Equal(addr) { - t.Fatalf("expecting flagged address to be blocked") - } - if len(blockedC) != 0 { - t.Fatalf("address should only be blocked once") - } + if a := <-blockedC; !a.Equal(addr) { + t.Fatalf("expecting flagged address to be blocked") + } + if len(blockedC) != 0 { + t.Fatalf("address should only be blocked once") + } + }) } func TestUnflagBeforeBlock(t *testing.T) { - t.Parallel() + synctest.Test(t, func(t *testing.T) { + addr := swarm.RandAddress(t) + mock := mockBlockLister(func(a swarm.Address, d time.Duration, r string) error { + t.Fatalf("address should not be blocked") - addr := swarm.RandAddress(t) - mock := mockBlockLister(func(a swarm.Address, d time.Duration, r string) error { - t.Fatalf("address should not be blocked") - - return nil - }) + return nil + }) - b := blocker.New(mock, flagTime, blockTime, time.Millisecond, nil, log.Noop) - testutil.CleanupCloser(t, b) + b := blocker.New(mock, flagTime, blockTime, time.Millisecond, nil, log.Noop) + testutil.CleanupCloser(t, b) - // Flagging address shouldn't block it imidietly - b.Flag(addr) + // Flagging address shouldn't block it immediately + b.Flag(addr) - time.Sleep(flagTime / 2) - b.Flag(addr) // check that this flag call does not override previous call + synctest.Wait() + b.Flag(addr) // check that this flag call does not override previous call - b.Unflag(addr) + b.Unflag(addr) - // Suspending current goroutine and expect that in this interval - // block listener was not called to block flagged address - time.Sleep(flagTime * 3) + // Suspending current goroutine and expect that in this interval + // block listener was not called to block flagged address + synctest.Wait() + }) } func TestPruneBeforeBlock(t *testing.T) { - t.Parallel() - - addr := swarm.RandAddress(t) - mock := mockBlockLister(func(a swarm.Address, d time.Duration, r string) error { - t.Fatalf("address should not be blocked") + synctest.Test(t, func(t *testing.T) { + addr := swarm.RandAddress(t) + mock := mockBlockLister(func(a swarm.Address, d time.Duration, r string) error { + t.Fatalf("address should not be blocked") - return nil - }) + return nil + }) - b := blocker.New(mock, flagTime, blockTime, time.Millisecond, nil, log.Noop) - testutil.CleanupCloser(t, b) + b := blocker.New(mock, flagTime, blockTime, time.Millisecond, nil, log.Noop) + testutil.CleanupCloser(t, b) - // Flagging address shouldn't block it imidietly - b.Flag(addr) + // Flagging address shouldn't block it immediately + b.Flag(addr) - time.Sleep(flagTime / 2) + synctest.Wait() - // communicate that we have seen no peers, resulting in the peer being removed - b.PruneUnseen([]swarm.Address{}) + // communicate that we have seen no peers, resulting in the peer being removed + b.PruneUnseen([]swarm.Address{}) - // Suspending current goroutine expect that in this interval - // block listener was not called to block flagged address - time.Sleep(flagTime * 3) + // Suspending current goroutine expect that in this interval + // block listener was not called to block flagged address + synctest.Wait() + }) } type blocklister struct { From 798006573e72b9523452002af1295908f0272aee Mon Sep 17 00:00:00 2001 From: Akrem Chabchoub <121046693+akrem-chabchoub@users.noreply.github.com> Date: Sun, 23 Nov 2025 22:50:21 +0100 Subject: [PATCH 50/96] test(synctest): add synctest to syncutil test (#5259) --- pkg/util/syncutil/syncutil_test.go | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/pkg/util/syncutil/syncutil_test.go b/pkg/util/syncutil/syncutil_test.go index ce05cec2dab..c73ed4cd980 100644 --- a/pkg/util/syncutil/syncutil_test.go +++ b/pkg/util/syncutil/syncutil_test.go @@ -7,23 +7,26 @@ package syncutil import ( "sync" "testing" + "testing/synctest" "time" ) func TestWaitWithTimeout(t *testing.T) { var wg sync.WaitGroup - if !WaitWithTimeout(&wg, 10*time.Millisecond) { - t.Fatal("want timeout; have none") - } + synctest.Test(t, func(t *testing.T) { + if !WaitWithTimeout(&wg, 10*time.Millisecond) { + t.Fatal("want timeout; have none") + } - wg.Add(1) - if WaitWithTimeout(&wg, 10*time.Millisecond) { - t.Fatal("have timeout; want none") - } + wg.Add(1) + if WaitWithTimeout(&wg, 10*time.Millisecond) { + t.Fatal("have timeout; want none") + } - wg.Done() - if !WaitWithTimeout(&wg, 10*time.Millisecond) { - t.Fatal("want no timeout; have none") - } + wg.Done() + if !WaitWithTimeout(&wg, 10*time.Millisecond) { + t.Fatal("want no timeout; have none") + } + }) } From 3298411a3227647e0f12ed3f73a5fee8950a556f Mon Sep 17 00:00:00 2001 From: Calin M Date: Tue, 25 Nov 2025 14:10:34 +0200 Subject: [PATCH 51/96] fix: joiner redundancy multilevel (#5292) --- pkg/file/joiner/joiner_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/file/joiner/joiner_test.go b/pkg/file/joiner/joiner_test.go index a8087a859db..10035aa2220 100644 --- a/pkg/file/joiner/joiner_test.go +++ b/pkg/file/joiner/joiner_test.go @@ -1357,6 +1357,9 @@ func TestJoinerRedundancyMultilevel(t *testing.T) { continue } t.Run(fmt.Sprintf("encrypt=%v levels=%d chunks=%d full", encrypt, levels, chunkCnt), func(t *testing.T) { + if r2level[rLevel] != levels || encrypt != encryptChunk[rLevel] { + t.Skip("skipping to save time") + } test(t, rLevel, encrypt, chunkCnt) }) } From cd6c34abae6b7e33f72957b0ffdd83943d67ba36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jano=C5=A1=20Gulja=C5=A1?= Date: Tue, 25 Nov 2025 22:27:17 +0100 Subject: [PATCH 52/96] fix: use hex encoded values in stamerindex.Item.String() (#5294) --- pkg/storer/internal/stampindex/stampindex.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/storer/internal/stampindex/stampindex.go b/pkg/storer/internal/stampindex/stampindex.go index b4d70839731..528fd998251 100644 --- a/pkg/storer/internal/stampindex/stampindex.go +++ b/pkg/storer/internal/stampindex/stampindex.go @@ -138,7 +138,7 @@ func (i *Item) Clone() storage.Item { // String implements the fmt.Stringer interface. func (i Item) String() string { - return storageutil.JoinFields(i.Namespace(), i.ID()) + return storageutil.JoinFields(i.Namespace(), fmt.Sprintf("%s/%x/%x", string(i.scope), i.BatchID, i.StampIndex)) } // LoadOrStore tries to first load a stamp index related record from the store. From 44d5d4fa54fd7bbf57a6d6350cce7d081ef4575f Mon Sep 17 00:00:00 2001 From: Akrem Chabchoub <121046693+akrem-chabchoub@users.noreply.github.com> Date: Sat, 29 Nov 2025 16:18:38 +0100 Subject: [PATCH 53/96] test(events): add synctest to subscriber tests (#5266) --- pkg/storer/internal/events/subscribe_test.go | 89 ++++++++++---------- 1 file changed, 45 insertions(+), 44 deletions(-) diff --git a/pkg/storer/internal/events/subscribe_test.go b/pkg/storer/internal/events/subscribe_test.go index 3e99494cd06..f4adefaea2f 100644 --- a/pkg/storer/internal/events/subscribe_test.go +++ b/pkg/storer/internal/events/subscribe_test.go @@ -6,54 +6,55 @@ package events_test import ( "testing" + "testing/synctest" "time" "github.com/ethersphere/bee/v2/pkg/storer/internal/events" ) func TestSubscriber(t *testing.T) { - t.Parallel() - - s := events.NewSubscriber() - - bin0_1, unsub0_1 := s.Subscribe("0") - bin0_2, unsub0_2 := s.Subscribe("0") - t.Cleanup(func() { unsub0_1(); unsub0_2() }) - go s.Trigger("0") - - gotSignals := make(chan struct{}) - - go func() { - defer close(gotSignals) - <-bin0_1 - <-bin0_2 - }() - - select { - case <-gotSignals: - case <-time.After(time.Second): - t.Fatal("signals did not fire in time") - } - - select { - case <-bin0_1: - t.Fatalf("trigger should not have fired again") - case <-bin0_2: - t.Fatalf("trigger should not have fired again") - default: - } - - bin1, unsub1 := s.Subscribe("1") - go s.Trigger("1") - go s.Trigger("1") - <-bin1 - <-bin1 - - unsub1() - - select { - case <-bin1: - t.Fatalf("trigger should not have fired again") - default: - } + synctest.Test(t, func(t *testing.T) { + s := events.NewSubscriber() + + bin0_1, unsub0_1 := s.Subscribe("0") + bin0_2, unsub0_2 := s.Subscribe("0") + t.Cleanup(func() { unsub0_1(); unsub0_2() }) + go s.Trigger("0") + + gotSignals := make(chan struct{}) + + go func() { + defer close(gotSignals) + <-bin0_1 + <-bin0_2 + }() + + select { + case <-gotSignals: + case <-time.After(time.Second): + t.Fatal("signals did not fire in time") + } + + select { + case <-bin0_1: + t.Fatalf("trigger should not have fired again") + case <-bin0_2: + t.Fatalf("trigger should not have fired again") + default: + } + + bin1, unsub1 := s.Subscribe("1") + go s.Trigger("1") + go s.Trigger("1") + <-bin1 + <-bin1 + + unsub1() + + select { + case <-bin1: + t.Fatalf("trigger should not have fired again") + default: + } + }) } From 11f8c81cdd07ea544b67671d12175b3e11be1dc7 Mon Sep 17 00:00:00 2001 From: Akrem Chabchoub <121046693+akrem-chabchoub@users.noreply.github.com> Date: Sat, 29 Nov 2025 16:21:56 +0100 Subject: [PATCH 54/96] test(storer-reserve): add synctest in evict test (#5263) --- pkg/storer/internal/reserve/reserve_test.go | 110 ++++++++++---------- 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/pkg/storer/internal/reserve/reserve_test.go b/pkg/storer/internal/reserve/reserve_test.go index 43c2252d736..87f700a98c8 100644 --- a/pkg/storer/internal/reserve/reserve_test.go +++ b/pkg/storer/internal/reserve/reserve_test.go @@ -11,7 +11,7 @@ import ( "math" "math/rand" "testing" - "time" + "testing/synctest" "github.com/ethersphere/bee/v2/pkg/crypto" "github.com/ethersphere/bee/v2/pkg/log" @@ -531,77 +531,77 @@ func TestReplaceOldIndex(t *testing.T) { } func TestEvict(t *testing.T) { - t.Parallel() + synctest.Test(t, func(t *testing.T) { + baseAddr := swarm.RandAddress(t) - baseAddr := swarm.RandAddress(t) + ts := internal.NewInmemStorage() - ts := internal.NewInmemStorage() + chunksPerBatch := 50 + var chunks []swarm.Chunk + batches := []*postage.Batch{postagetesting.MustNewBatch(), postagetesting.MustNewBatch(), postagetesting.MustNewBatch()} + evictBatch := batches[1] - chunksPerBatch := 50 - var chunks []swarm.Chunk - batches := []*postage.Batch{postagetesting.MustNewBatch(), postagetesting.MustNewBatch(), postagetesting.MustNewBatch()} - evictBatch := batches[1] + r, err := reserve.New( + baseAddr, + ts, + 0, kademlia.NewTopologyDriver(), + log.Noop, + ) + if err != nil { + t.Fatal(err) + } - r, err := reserve.New( - baseAddr, - ts, - 0, kademlia.NewTopologyDriver(), - log.Noop, - ) - if err != nil { - t.Fatal(err) - } + for range chunksPerBatch { + for b := range 3 { + ch := chunk.GenerateTestRandomChunkAt(t, baseAddr, b).WithStamp(postagetesting.MustNewBatchStamp(batches[b].ID)) + chunks = append(chunks, ch) + err := r.Put(context.Background(), ch) + if err != nil { + t.Fatal(err) + } + } + } - for range chunksPerBatch { - for b := range 3 { - ch := chunk.GenerateTestRandomChunkAt(t, baseAddr, b).WithStamp(postagetesting.MustNewBatchStamp(batches[b].ID)) - chunks = append(chunks, ch) - err := r.Put(context.Background(), ch) + totalEvicted := 0 + for i := range 3 { + evicted, err := r.EvictBatchBin(context.Background(), evictBatch.ID, math.MaxInt, uint8(i)) if err != nil { t.Fatal(err) } + totalEvicted += evicted } - } - totalEvicted := 0 - for i := range 3 { - evicted, err := r.EvictBatchBin(context.Background(), evictBatch.ID, math.MaxInt, uint8(i)) - if err != nil { - t.Fatal(err) + if totalEvicted != chunksPerBatch { + t.Fatalf("got %d, want %d", totalEvicted, chunksPerBatch) } - totalEvicted += evicted - } - if totalEvicted != chunksPerBatch { - t.Fatalf("got %d, want %d", totalEvicted, chunksPerBatch) - } + synctest.Wait() - time.Sleep(time.Second) - - for i, ch := range chunks { - binID := i%chunksPerBatch + 1 - b := swarm.Proximity(baseAddr.Bytes(), ch.Address().Bytes()) - stampHash, err := ch.Stamp().Hash() - if err != nil { - t.Fatal(err) - } - _, err = r.Get(context.Background(), ch.Address(), ch.Stamp().BatchID(), stampHash) - if bytes.Equal(ch.Stamp().BatchID(), evictBatch.ID) { - if !errors.Is(err, storage.ErrNotFound) { - t.Fatalf("got err %v, want %v", err, storage.ErrNotFound) - } - checkStore(t, ts.IndexStore(), &reserve.BatchRadiusItem{Bin: b, BatchID: ch.Stamp().BatchID(), Address: ch.Address(), StampHash: stampHash}, true) - checkStore(t, ts.IndexStore(), &reserve.ChunkBinItem{Bin: b, BinID: uint64(binID), StampHash: stampHash}, true) - checkChunk(t, ts, ch, true) - } else { + for i, ch := range chunks { + binID := i%chunksPerBatch + 1 + b := swarm.Proximity(baseAddr.Bytes(), ch.Address().Bytes()) + stampHash, err := ch.Stamp().Hash() if err != nil { t.Fatal(err) } - checkStore(t, ts.IndexStore(), &reserve.BatchRadiusItem{Bin: b, BatchID: ch.Stamp().BatchID(), Address: ch.Address(), StampHash: stampHash}, false) - checkStore(t, ts.IndexStore(), &reserve.ChunkBinItem{Bin: b, BinID: uint64(binID), StampHash: stampHash}, false) - checkChunk(t, ts, ch, false) + _, err = r.Get(context.Background(), ch.Address(), ch.Stamp().BatchID(), stampHash) + if bytes.Equal(ch.Stamp().BatchID(), evictBatch.ID) { + if !errors.Is(err, storage.ErrNotFound) { + t.Fatalf("got err %v, want %v", err, storage.ErrNotFound) + } + checkStore(t, ts.IndexStore(), &reserve.BatchRadiusItem{Bin: b, BatchID: ch.Stamp().BatchID(), Address: ch.Address(), StampHash: stampHash}, true) + checkStore(t, ts.IndexStore(), &reserve.ChunkBinItem{Bin: b, BinID: uint64(binID), StampHash: stampHash}, true) + checkChunk(t, ts, ch, true) + } else { + if err != nil { + t.Fatal(err) + } + checkStore(t, ts.IndexStore(), &reserve.BatchRadiusItem{Bin: b, BatchID: ch.Stamp().BatchID(), Address: ch.Address(), StampHash: stampHash}, false) + checkStore(t, ts.IndexStore(), &reserve.ChunkBinItem{Bin: b, BinID: uint64(binID), StampHash: stampHash}, false) + checkChunk(t, ts, ch, false) + } } - } + }) } func TestEvictSOC(t *testing.T) { From 8e716cb845e28301ac4198f5f694364c76063675 Mon Sep 17 00:00:00 2001 From: Akrem Chabchoub <121046693+akrem-chabchoub@users.noreply.github.com> Date: Mon, 1 Dec 2025 19:30:33 +0100 Subject: [PATCH 55/96] Add synctest in storage incentives (#5262) --- pkg/storageincentives/events_test.go | 145 +++++++++++++------------ pkg/storageincentives/soc_mine_test.go | 104 +++++++++--------- 2 files changed, 126 insertions(+), 123 deletions(-) diff --git a/pkg/storageincentives/events_test.go b/pkg/storageincentives/events_test.go index f5a00d05c5b..685512ebd73 100644 --- a/pkg/storageincentives/events_test.go +++ b/pkg/storageincentives/events_test.go @@ -7,87 +7,88 @@ package storageincentives_test import ( "context" "testing" + "testing/synctest" "time" "github.com/ethersphere/bee/v2/pkg/storageincentives" ) func TestClose(t *testing.T) { - t.Parallel() - - ev := storageincentives.NewEvents() - - done1 := make(chan struct{}) - done2 := make(chan struct{}) - done3 := make(chan struct{}) - - ev.On(1, func(ctx context.Context) { - <-ctx.Done() - close(done1) - }) - - ev.On(1, func(ctx context.Context) { - <-ctx.Done() - close(done2) - }) - - ev.On(2, func(ctx context.Context) { - <-ctx.Done() - close(done3) - }) - - ev.Publish(1) - ev.Publish(2) - - ev.Close() - - for range 3 { - select { - case <-done1: - case <-done2: - case <-done3: - case <-time.After(time.Second): - t.Fatal("timeout") + synctest.Test(t, func(t *testing.T) { + ev := storageincentives.NewEvents() + + done1 := make(chan struct{}) + done2 := make(chan struct{}) + done3 := make(chan struct{}) + + ev.On(1, func(ctx context.Context) { + <-ctx.Done() + close(done1) + }) + + ev.On(1, func(ctx context.Context) { + <-ctx.Done() + close(done2) + }) + + ev.On(2, func(ctx context.Context) { + <-ctx.Done() + close(done3) + }) + + ev.Publish(1) + ev.Publish(2) + + ev.Close() + + for range 3 { + select { + case <-done1: + case <-done2: + case <-done3: + case <-time.After(time.Second): + t.Fatal("timeout") + } } - } + }) } func TestPhaseCancel(t *testing.T) { - t.Parallel() - - ev := storageincentives.NewEvents() - - done1 := make(chan struct{}) - done2 := make(chan struct{}) - defer ev.Close() - - // ensure no panics occur on an empty publish - ev.Publish(0) - - ev.On(1, func(ctx context.Context) { - <-ctx.Done() - close(done1) - }) - - ev.On(2, func(ctx context.Context) { - <-ctx.Done() - close(done2) - }) - - ev.On(3, func(ctx context.Context) { - ev.Cancel(1, 2) - }) - - ev.Publish(1) - ev.Publish(2) - ev.Publish(3) - - for range 2 { - select { - case <-done1: - case <-done2: - case <-time.After(time.Second): - t.Fatal("timeout") + synctest.Test(t, func(t *testing.T) { + ev := storageincentives.NewEvents() + + done1 := make(chan struct{}) + done2 := make(chan struct{}) + defer ev.Close() + + // ensure no panics occur on an empty publish + ev.Publish(0) + + ev.On(1, func(ctx context.Context) { + <-ctx.Done() + close(done1) + }) + + ev.On(2, func(ctx context.Context) { + <-ctx.Done() + close(done2) + }) + + ev.On(3, func(ctx context.Context) { + ev.Cancel(1, 2) + }) + + ev.Publish(1) + ev.Publish(2) + ev.Publish(3) + + for range 2 { + select { + case <-done1: + case <-done2: + case <-time.After(time.Second): + t.Fatal("timeout") + } } - } + }) } diff --git a/pkg/storageincentives/soc_mine_test.go b/pkg/storageincentives/soc_mine_test.go index 4f3cd7812e8..0265a9a21f7 100644 --- a/pkg/storageincentives/soc_mine_test.go +++ b/pkg/storageincentives/soc_mine_test.go @@ -14,6 +14,7 @@ import ( "os" "sync" "testing" + "testing/synctest" "github.com/ethersphere/bee/v2/pkg/bmt" "github.com/ethersphere/bee/v2/pkg/cac" @@ -32,58 +33,59 @@ import ( // to generate uploads using the input // cat socs.txt | tail 19 | head 16 | perl -pne 's/([a-f0-9]+)\t([a-f0-9]+)\t([a-f0-9]+)\t([a-f0-9]+)/echo -n $4 | xxd -r -p | curl -X POST \"http:\/\/localhost:1633\/soc\/$1\/$2?sig=$3\" -H \"accept: application\/json, text\/plain, \/\" -H \"content-type: application\/octet-stream\" -H \"swarm-postage-batch-id: 14b26beca257e763609143c6b04c2c487f01a051798c535c2f542ce75a97c05f\" --data-binary \@-/' func TestSocMine(t *testing.T) { - t.Parallel() - // the anchor used in neighbourhood selection and reserve salt for sampling - prefix, err := hex.DecodeString("3617319a054d772f909f7c479a2cebe5066e836a939412e32403c99029b92eff") - if err != nil { - t.Fatal(err) - } - // the transformed address hasher factory function - prefixhasher := func() hash.Hash { return swarm.NewPrefixHasher(prefix) } - // Create a pool for efficient hasher reuse - trHasherPool := bmt.NewPool(bmt.NewConf(prefixhasher, swarm.BmtBranches, 8)) - // the bignum cast of the maximum sample value (upper bound on transformed addresses as a 256-bit article) - // this constant is for a minimum reserve size of 2 million chunks with sample size of 16 - // = 1.284401 * 10^71 = 1284401 + 66 0-s - mstring := "1284401" - for range 66 { - mstring = mstring + "0" - } - n, ok := new(big.Int).SetString(mstring, 10) - if !ok { - t.Fatalf("SetString: error setting to '%s'", mstring) - } - // the filter function on the SOC address - // meant to make sure we pass check for proof of retrievability for - // a node of overlay 0x65xxx with a reserve depth of 1, i.e., - // SOC address must start with zero bit - filterSOCAddr := func(a swarm.Address) bool { - return a.Bytes()[0]&0x80 != 0x00 - } - // the filter function on the transformed address using the density estimation constant - filterTrAddr := func(a swarm.Address) (bool, error) { - m := new(big.Int).SetBytes(a.Bytes()) - return m.Cmp(n) < 0, nil - } - // setup the signer with a private key from a fixture - data, err := hex.DecodeString("634fb5a872396d9693e5c9f9d7233cfa93f395c093371017ff44aa9ae6564cdd") - if err != nil { - t.Fatal(err) - } - privKey, err := crypto.DecodeSecp256k1PrivateKey(data) - if err != nil { - t.Fatal(err) - } - signer := crypto.NewDefaultSigner(privKey) + synctest.Test(t, func(t *testing.T) { + // the anchor used in neighbourhood selection and reserve salt for sampling + prefix, err := hex.DecodeString("3617319a054d772f909f7c479a2cebe5066e836a939412e32403c99029b92eff") + if err != nil { + t.Fatal(err) + } + // the transformed address hasher factory function + prefixhasher := func() hash.Hash { return swarm.NewPrefixHasher(prefix) } + // Create a pool for efficient hasher reuse + trHasherPool := bmt.NewPool(bmt.NewConf(prefixhasher, swarm.BmtBranches, 8)) + // the bignum cast of the maximum sample value (upper bound on transformed addresses as a 256-bit article) + // this constant is for a minimum reserve size of 2 million chunks with sample size of 16 + // = 1.284401 * 10^71 = 1284401 + 66 0-s + mstring := "1284401" + for range 66 { + mstring = mstring + "0" + } + n, ok := new(big.Int).SetString(mstring, 10) + if !ok { + t.Fatalf("SetString: error setting to '%s'", mstring) + } + // the filter function on the SOC address + // meant to make sure we pass check for proof of retrievability for + // a node of overlay 0x65xxx with a reserve depth of 1, i.e., + // SOC address must start with zero bit + filterSOCAddr := func(a swarm.Address) bool { + return a.Bytes()[0]&0x80 != 0x00 + } + // the filter function on the transformed address using the density estimation constant + filterTrAddr := func(a swarm.Address) (bool, error) { + m := new(big.Int).SetBytes(a.Bytes()) + return m.Cmp(n) < 0, nil + } + // setup the signer with a private key from a fixture + data, err := hex.DecodeString("634fb5a872396d9693e5c9f9d7233cfa93f395c093371017ff44aa9ae6564cdd") + if err != nil { + t.Fatal(err) + } + privKey, err := crypto.DecodeSecp256k1PrivateKey(data) + if err != nil { + t.Fatal(err) + } + signer := crypto.NewDefaultSigner(privKey) - sampleSize := 16 - // for sanity check: given a filterSOCAddr requiring a 0 leading bit (chance of 1/2) - // we expect an overall rough 4 million chunks to be mined to create this sample - // for 8 workers that is half a million round on average per worker - err = makeChunks(t, signer, sampleSize, filterSOCAddr, filterTrAddr, trHasherPool) - if err != nil { - t.Fatal(err) - } + sampleSize := 16 + // for sanity check: given a filterSOCAddr requiring a 0 leading bit (chance of 1/2) + // we expect an overall rough 4 million chunks to be mined to create this sample + // for 8 workers that is half a million round on average per worker + err = makeChunks(t, signer, sampleSize, filterSOCAddr, filterTrAddr, trHasherPool) + if err != nil { + t.Fatal(err) + } + }) } func makeChunks(t *testing.T, signer crypto.Signer, sampleSize int, filterSOCAddr func(swarm.Address) bool, filterTrAddr func(swarm.Address) (bool, error), trHasherPool *bmt.Pool) error { From aed9d558bedca84d67bfb50635d8d5c097741cc9 Mon Sep 17 00:00:00 2001 From: Akrem Chabchoub <121046693+akrem-chabchoub@users.noreply.github.com> Date: Mon, 1 Dec 2025 19:53:39 +0100 Subject: [PATCH 56/96] test(pullsync): add synctest to pullsync (#5261) --- pkg/pullsync/pullsync_test.go | 413 +++++++++++++++++----------------- 1 file changed, 207 insertions(+), 206 deletions(-) diff --git a/pkg/pullsync/pullsync_test.go b/pkg/pullsync/pullsync_test.go index d77b130936e..03d6927c1da 100644 --- a/pkg/pullsync/pullsync_test.go +++ b/pkg/pullsync/pullsync_test.go @@ -9,6 +9,7 @@ import ( "errors" "io" "testing" + "testing/synctest" "time" "github.com/ethersphere/bee/v2/pkg/log" @@ -58,256 +59,256 @@ func init() { } func TestIncoming_WantNone(t *testing.T) { - t.Parallel() - - var ( - topMost = uint64(4) - ps, _ = newPullSync(t, nil, 5, mock.WithSubscribeResp(results, nil), mock.WithChunks(chunks...)) - recorder = streamtest.New(streamtest.WithProtocols(ps.Protocol())) - psClient, clientDb = newPullSync(t, recorder, 0, mock.WithChunks(chunks...)) - ) - - topmost, _, err := psClient.Sync(context.Background(), swarm.ZeroAddress, 0, 0) - if err != nil { - t.Fatal(err) - } + synctest.Test(t, func(t *testing.T) { + var ( + topMost = uint64(4) + ps, _ = newPullSync(t, nil, 5, mock.WithSubscribeResp(results, nil), mock.WithChunks(chunks...)) + recorder = streamtest.New(streamtest.WithProtocols(ps.Protocol())) + psClient, clientDb = newPullSync(t, recorder, 0, mock.WithChunks(chunks...)) + ) + + topmost, _, err := psClient.Sync(context.Background(), swarm.ZeroAddress, 0, 0) + if err != nil { + t.Fatal(err) + } - if topmost != topMost { - t.Fatalf("got offer topmost %d but want %d", topmost, topMost) - } - if clientDb.PutCalls() > 0 { - t.Fatal("too many puts") - } + if topmost != topMost { + t.Fatalf("got offer topmost %d but want %d", topmost, topMost) + } + if clientDb.PutCalls() > 0 { + t.Fatal("too many puts") + } + }) } func TestIncoming_ContextTimeout(t *testing.T) { - t.Parallel() - - var ( - ps, _ = newPullSync(t, nil, 0, mock.WithSubscribeResp(results, nil), mock.WithChunks(chunks...)) - recorder = streamtest.New(streamtest.WithProtocols(ps.Protocol())) - psClient, _ = newPullSync(t, recorder, 0, mock.WithChunks(chunks...)) - ) - - ctx, cancel := context.WithTimeout(context.Background(), 0) - cancel() - _, _, err := psClient.Sync(ctx, swarm.ZeroAddress, 0, 0) - if !errors.Is(err, context.DeadlineExceeded) { - t.Fatalf("wanted error %v, got %v", context.DeadlineExceeded, err) - } + synctest.Test(t, func(t *testing.T) { + var ( + ps, _ = newPullSync(t, nil, 0, mock.WithSubscribeResp(results, nil), mock.WithChunks(chunks...)) + recorder = streamtest.New(streamtest.WithProtocols(ps.Protocol())) + psClient, _ = newPullSync(t, recorder, 0, mock.WithChunks(chunks...)) + ) + + ctx, cancel := context.WithTimeout(context.Background(), 0) + cancel() + _, _, err := psClient.Sync(ctx, swarm.ZeroAddress, 0, 0) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("wanted error %v, got %v", context.DeadlineExceeded, err) + } + }) } func TestIncoming_WantOne(t *testing.T) { - t.Parallel() - - var ( - topMost = uint64(4) - ps, _ = newPullSync(t, nil, 5, mock.WithSubscribeResp(results, nil), mock.WithChunks(chunks...)) - recorder = streamtest.New(streamtest.WithProtocols(ps.Protocol())) - psClient, clientDb = newPullSync(t, recorder, 0, mock.WithChunks(someChunks(1, 2, 3, 4)...)) - ) - - topmost, _, err := psClient.Sync(context.Background(), swarm.ZeroAddress, 0, 0) - if err != nil { - t.Fatal(err) - } + synctest.Test(t, func(t *testing.T) { + var ( + topMost = uint64(4) + ps, _ = newPullSync(t, nil, 5, mock.WithSubscribeResp(results, nil), mock.WithChunks(chunks...)) + recorder = streamtest.New(streamtest.WithProtocols(ps.Protocol())) + psClient, clientDb = newPullSync(t, recorder, 0, mock.WithChunks(someChunks(1, 2, 3, 4)...)) + ) + + topmost, _, err := psClient.Sync(context.Background(), swarm.ZeroAddress, 0, 0) + if err != nil { + t.Fatal(err) + } - if topmost != topMost { - t.Fatalf("got offer topmost %d but want %d", topmost, topMost) - } + if topmost != topMost { + t.Fatalf("got offer topmost %d but want %d", topmost, topMost) + } - // should have all - haveChunks(t, clientDb, chunks...) - if clientDb.PutCalls() != 1 { - t.Fatalf("want 1 puts but got %d", clientDb.PutCalls()) - } + // should have all + haveChunks(t, clientDb, chunks...) + if clientDb.PutCalls() != 1 { + t.Fatalf("want 1 puts but got %d", clientDb.PutCalls()) + } + }) } func TestIncoming_WantAll(t *testing.T) { - t.Parallel() - - var ( - topMost = uint64(4) - ps, _ = newPullSync(t, nil, 5, mock.WithSubscribeResp(results, nil), mock.WithChunks(chunks...)) - recorder = streamtest.New(streamtest.WithProtocols(ps.Protocol())) - psClient, clientDb = newPullSync(t, recorder, 0) - ) - - topmost, _, err := psClient.Sync(context.Background(), swarm.ZeroAddress, 0, 0) - if err != nil { - t.Fatal(err) - } + synctest.Test(t, func(t *testing.T) { + var ( + topMost = uint64(4) + ps, _ = newPullSync(t, nil, 5, mock.WithSubscribeResp(results, nil), mock.WithChunks(chunks...)) + recorder = streamtest.New(streamtest.WithProtocols(ps.Protocol())) + psClient, clientDb = newPullSync(t, recorder, 0) + ) + + topmost, _, err := psClient.Sync(context.Background(), swarm.ZeroAddress, 0, 0) + if err != nil { + t.Fatal(err) + } - if topmost != topMost { - t.Fatalf("got offer topmost %d but want %d", topmost, topMost) - } + if topmost != topMost { + t.Fatalf("got offer topmost %d but want %d", topmost, topMost) + } - // should have all - haveChunks(t, clientDb, chunks...) - if p := clientDb.PutCalls(); p != len(chunks) { - t.Fatalf("want %d puts but got %d", len(chunks), p) - } + // should have all + haveChunks(t, clientDb, chunks...) + if p := clientDb.PutCalls(); p != len(chunks) { + t.Fatalf("want %d puts but got %d", len(chunks), p) + } + }) } func TestIncoming_WantErrors(t *testing.T) { - t.Parallel() - - tChunks := testingc.GenerateTestRandomChunks(4) - // add same chunk with a different batch id - ch := swarm.NewChunk(tChunks[3].Address(), tChunks[3].Data()).WithStamp(postagetesting.MustNewStamp()) - tChunks = append(tChunks, ch) - // add invalid chunk - tChunks = append(tChunks, testingc.GenerateTestRandomInvalidChunk()) - - tResults := make([]*storer.BinC, len(tChunks)) - for i, c := range tChunks { - stampHash, err := c.Stamp().Hash() - if err != nil { - t.Fatal(err) + synctest.Test(t, func(t *testing.T) { + tChunks := testingc.GenerateTestRandomChunks(4) + // add same chunk with a different batch id + ch := swarm.NewChunk(tChunks[3].Address(), tChunks[3].Data()).WithStamp(postagetesting.MustNewStamp()) + tChunks = append(tChunks, ch) + // add invalid chunk + tChunks = append(tChunks, testingc.GenerateTestRandomInvalidChunk()) + + tResults := make([]*storer.BinC, len(tChunks)) + for i, c := range tChunks { + stampHash, err := c.Stamp().Hash() + if err != nil { + t.Fatal(err) + } + tResults[i] = &storer.BinC{ + Address: c.Address(), + BatchID: c.Stamp().BatchID(), + BinID: uint64(i + 5), // start from a higher bin id + StampHash: stampHash, + } } - tResults[i] = &storer.BinC{ - Address: c.Address(), - BatchID: c.Stamp().BatchID(), - BinID: uint64(i + 5), // start from a higher bin id - StampHash: stampHash, - } - } - putHook := func(c swarm.Chunk) error { - if c.Address().Equal(tChunks[1].Address()) { - return storage.ErrOverwriteNewerChunk + putHook := func(c swarm.Chunk) error { + if c.Address().Equal(tChunks[1].Address()) { + return storage.ErrOverwriteNewerChunk + } + return nil } - return nil - } - validStampErr := errors.New("valid stamp error") - validStamp := func(c swarm.Chunk) (swarm.Chunk, error) { - if c.Address().Equal(tChunks[2].Address()) { - return nil, validStampErr + validStampErr := errors.New("valid stamp error") + validStamp := func(c swarm.Chunk) (swarm.Chunk, error) { + if c.Address().Equal(tChunks[2].Address()) { + return nil, validStampErr + } + return c, nil } - return c, nil - } - - var ( - topMost = uint64(10) - ps, _ = newPullSync(t, nil, 20, mock.WithSubscribeResp(tResults, nil), mock.WithChunks(tChunks...)) - recorder = streamtest.New(streamtest.WithProtocols(ps.Protocol())) - psClient, clientDb = newPullSyncWithStamperValidator(t, recorder, 0, validStamp, mock.WithPutHook(putHook)) - ) - topmost, count, err := psClient.Sync(context.Background(), swarm.ZeroAddress, 0, 0) - for _, e := range []error{storage.ErrOverwriteNewerChunk, validStampErr, swarm.ErrInvalidChunk} { - if !errors.Is(err, e) { - t.Fatalf("expected error %v", err) + var ( + topMost = uint64(10) + ps, _ = newPullSync(t, nil, 20, mock.WithSubscribeResp(tResults, nil), mock.WithChunks(tChunks...)) + recorder = streamtest.New(streamtest.WithProtocols(ps.Protocol())) + psClient, clientDb = newPullSyncWithStamperValidator(t, recorder, 0, validStamp, mock.WithPutHook(putHook)) + ) + + topmost, count, err := psClient.Sync(context.Background(), swarm.ZeroAddress, 0, 0) + for _, e := range []error{storage.ErrOverwriteNewerChunk, validStampErr, swarm.ErrInvalidChunk} { + if !errors.Is(err, e) { + t.Fatalf("expected error %v", err) + } } - } - if count != 3 { - t.Fatalf("got %d chunks but want %d", count, 3) - } + if count != 3 { + t.Fatalf("got %d chunks but want %d", count, 3) + } - if topmost != topMost { - t.Fatalf("got offer topmost %d but want %d", topmost, topMost) - } + if topmost != topMost { + t.Fatalf("got offer topmost %d but want %d", topmost, topMost) + } - haveChunks(t, clientDb, append(tChunks[:1], tChunks[3:5]...)...) - if p := clientDb.PutCalls(); p != len(chunks)-1 { - t.Fatalf("want %d puts but got %d", len(chunks), p) - } + haveChunks(t, clientDb, append(tChunks[:1], tChunks[3:5]...)...) + if p := clientDb.PutCalls(); p != len(chunks)-1 { + t.Fatalf("want %d puts but got %d", len(chunks), p) + } + }) } func TestIncoming_UnsolicitedChunk(t *testing.T) { - t.Parallel() - - evilAddr := swarm.MustParseHexAddress("0000000000000000000000000000000000000000000000000000000000000666") - evilData := []byte{0x66, 0x66, 0x66} - stamp := postagetesting.MustNewStamp() - evil := swarm.NewChunk(evilAddr, evilData).WithStamp(stamp) - - var ( - ps, _ = newPullSync(t, nil, 5, mock.WithSubscribeResp(results, nil), mock.WithChunks(chunks...), mock.WithEvilChunk(addrs[4], evil)) - recorder = streamtest.New(streamtest.WithProtocols(ps.Protocol())) - psClient, _ = newPullSync(t, recorder, 0) - ) - - _, _, err := psClient.Sync(context.Background(), swarm.ZeroAddress, 0, 0) - if !errors.Is(err, pullsync.ErrUnsolicitedChunk) { - t.Fatalf("expected err %v but got %v", pullsync.ErrUnsolicitedChunk, err) - } + synctest.Test(t, func(t *testing.T) { + evilAddr := swarm.MustParseHexAddress("0000000000000000000000000000000000000000000000000000000000000666") + evilData := []byte{0x66, 0x66, 0x66} + stamp := postagetesting.MustNewStamp() + evil := swarm.NewChunk(evilAddr, evilData).WithStamp(stamp) + + var ( + ps, _ = newPullSync(t, nil, 5, mock.WithSubscribeResp(results, nil), mock.WithChunks(chunks...), mock.WithEvilChunk(addrs[4], evil)) + recorder = streamtest.New(streamtest.WithProtocols(ps.Protocol())) + psClient, _ = newPullSync(t, recorder, 0) + ) + + _, _, err := psClient.Sync(context.Background(), swarm.ZeroAddress, 0, 0) + if !errors.Is(err, pullsync.ErrUnsolicitedChunk) { + t.Fatalf("expected err %v but got %v", pullsync.ErrUnsolicitedChunk, err) + } + }) } func TestMissingChunk(t *testing.T) { - t.Parallel() - - var ( - zeroChunk = swarm.NewChunk(swarm.ZeroAddress, nil) - topMost = uint64(4) - ps, _ = newPullSync(t, nil, 5, mock.WithSubscribeResp(results, nil), mock.WithChunks([]swarm.Chunk{zeroChunk}...)) - recorder = streamtest.New(streamtest.WithProtocols(ps.Protocol())) - psClient, _ = newPullSync(t, recorder, 0) - ) - - topmost, count, err := psClient.Sync(context.Background(), swarm.ZeroAddress, 0, 0) - if err != nil { - t.Fatal(err) - } + synctest.Test(t, func(t *testing.T) { + var ( + zeroChunk = swarm.NewChunk(swarm.ZeroAddress, nil) + topMost = uint64(4) + ps, _ = newPullSync(t, nil, 5, mock.WithSubscribeResp(results, nil), mock.WithChunks([]swarm.Chunk{zeroChunk}...)) + recorder = streamtest.New(streamtest.WithProtocols(ps.Protocol())) + psClient, _ = newPullSync(t, recorder, 0) + ) + + topmost, count, err := psClient.Sync(context.Background(), swarm.ZeroAddress, 0, 0) + if err != nil { + t.Fatal(err) + } - if topmost != topMost { - t.Fatalf("got offer topmost %d but want %d", topmost, topMost) - } - if count != 0 { - t.Fatalf("got count %d but want %d", count, 0) - } + if topmost != topMost { + t.Fatalf("got offer topmost %d but want %d", topmost, topMost) + } + if count != 0 { + t.Fatalf("got count %d but want %d", count, 0) + } + }) } func TestGetCursors(t *testing.T) { - t.Parallel() - - var ( - epochTs = uint64(time.Now().Unix()) - mockCursors = []uint64{100, 101, 102, 103} - ps, _ = newPullSync(t, nil, 0, mock.WithCursors(mockCursors, epochTs)) - recorder = streamtest.New(streamtest.WithProtocols(ps.Protocol())) - psClient, _ = newPullSync(t, recorder, 0) - ) - - curs, epoch, err := psClient.GetCursors(context.Background(), swarm.ZeroAddress) - if err != nil { - t.Fatal(err) - } + synctest.Test(t, func(t *testing.T) { + var ( + epochTs = uint64(time.Now().Unix()) + mockCursors = []uint64{100, 101, 102, 103} + ps, _ = newPullSync(t, nil, 0, mock.WithCursors(mockCursors, epochTs)) + recorder = streamtest.New(streamtest.WithProtocols(ps.Protocol())) + psClient, _ = newPullSync(t, recorder, 0) + ) + + curs, epoch, err := psClient.GetCursors(context.Background(), swarm.ZeroAddress) + if err != nil { + t.Fatal(err) + } - if len(curs) != len(mockCursors) { - t.Fatalf("length mismatch got %d want %d", len(curs), len(mockCursors)) - } + if len(curs) != len(mockCursors) { + t.Fatalf("length mismatch got %d want %d", len(curs), len(mockCursors)) + } - if epochTs != epoch { - t.Fatalf("epochs do not match got %d want %d", epoch, epochTs) - } + if epochTs != epoch { + t.Fatalf("epochs do not match got %d want %d", epoch, epochTs) + } - for i, v := range mockCursors { - if curs[i] != v { - t.Errorf("cursor mismatch. index %d want %d got %d", i, v, curs[i]) + for i, v := range mockCursors { + if curs[i] != v { + t.Errorf("cursor mismatch. index %d want %d got %d", i, v, curs[i]) + } } - } + }) } func TestGetCursorsError(t *testing.T) { - t.Parallel() - - var ( - e = errors.New("erring") - ps, _ = newPullSync(t, nil, 0, mock.WithCursorsErr(e)) - recorder = streamtest.New(streamtest.WithProtocols(ps.Protocol())) - psClient, _ = newPullSync(t, recorder, 0) - ) - - _, _, err := psClient.GetCursors(context.Background(), swarm.ZeroAddress) - if err == nil { - t.Fatal("expected error but got none") - } - if !errors.Is(err, io.EOF) { - t.Fatalf("expect error '%v' but got '%v'", e, err) - } + synctest.Test(t, func(t *testing.T) { + var ( + e = errors.New("erring") + ps, _ = newPullSync(t, nil, 0, mock.WithCursorsErr(e)) + recorder = streamtest.New(streamtest.WithProtocols(ps.Protocol())) + psClient, _ = newPullSync(t, recorder, 0) + ) + + _, _, err := psClient.GetCursors(context.Background(), swarm.ZeroAddress) + if err == nil { + t.Fatal("expected error but got none") + } + if !errors.Is(err, io.EOF) { + t.Fatalf("expect error '%v' but got '%v'", e, err) + } + }) } func haveChunks(t *testing.T, s *mock.ReserveStore, chunks ...swarm.Chunk) { From 492ea5336f326019f87dfa1a7b2f12ebc931a3d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ljubi=C5=A1a=20Ga=C4=8Devi=C4=87?= <35105035+gacevicljubisa@users.noreply.github.com> Date: Tue, 2 Dec 2025 16:24:02 +0100 Subject: [PATCH 57/96] feat(pushsync): add rate limiting and metric for overdraft refresh logs (#5297) --- pkg/pushsync/metrics.go | 7 +++++++ pkg/pushsync/pushsync.go | 8 +++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/pkg/pushsync/metrics.go b/pkg/pushsync/metrics.go index 7f4f5884bf0..758a6f072a5 100644 --- a/pkg/pushsync/metrics.go +++ b/pkg/pushsync/metrics.go @@ -28,6 +28,7 @@ type metrics struct { ReceiptDepth *prometheus.CounterVec ShallowReceiptDepth *prometheus.CounterVec ShallowReceipt prometheus.Counter + OverdraftRefresh prometheus.Counter } func newMetrics() metrics { @@ -146,6 +147,12 @@ func newMetrics() metrics { }, []string{"depth"}, ), + OverdraftRefresh: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: m.Namespace, + Subsystem: subsystem, + Name: "overdraft_refresh", + Help: "Total number of times peers were skipped due to overdraft, requiring a wait to refresh balance.", + }), } } diff --git a/pkg/pushsync/pushsync.go b/pkg/pushsync/pushsync.go index 9a9754a9ab3..715680430cf 100644 --- a/pkg/pushsync/pushsync.go +++ b/pkg/pushsync/pushsync.go @@ -32,6 +32,7 @@ import ( "github.com/opentracing/opentracing-go" "github.com/opentracing/opentracing-go/ext" olog "github.com/opentracing/opentracing-go/log" + "golang.org/x/time/rate" ) // loggerName is the tree path name of the logger for this package. @@ -99,6 +100,7 @@ type PushSync struct { stabilizer stabilization.Subscriber shallowReceiptTolerance uint8 + overDraftRefreshLimiter *rate.Limiter } type receiptResult struct { @@ -148,6 +150,7 @@ func New( errSkip: skippeers.NewList(time.Minute), stabilizer: stabilizer, shallowReceiptTolerance: shallowReceiptTolerance, + overDraftRefreshLimiter: rate.NewLimiter(rate.Every(time.Second), 1), } ps.validStamp = ps.validStampWrapper(validStamp) @@ -424,7 +427,10 @@ func (ps *PushSync) pushToClosest(ctx context.Context, ch swarm.Chunk, origin bo continue // there is still an inflight request, wait for it's result } - ps.logger.Debug("sleeping to refresh overdraft balance", "chunk_address", ch.Address()) + ps.metrics.OverdraftRefresh.Inc() + if ps.overDraftRefreshLimiter.Allow() { + ps.logger.Debug("sleeping to refresh overdraft balance") + } select { case <-time.After(overDraftRefresh): From 7a72d9f9aedaabd80dd56f91237dbe93474635f8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 4 Dec 2025 14:21:30 +0100 Subject: [PATCH 58/96] chore(deps): bump golang.org/x/crypto from 0.36.0 to 0.45.0 (#5286) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 18 ++++++++++-------- go.sum | 34 ++++++++++++++++++---------------- 2 files changed, 28 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index adae83026a2..a0439b609ed 100644 --- a/go.mod +++ b/go.mod @@ -46,10 +46,10 @@ require ( gitlab.com/nolash/go-mockbytes v0.0.7 go.uber.org/atomic v1.11.0 go.uber.org/goleak v1.3.0 - golang.org/x/crypto v0.36.0 - golang.org/x/sync v0.12.0 - golang.org/x/sys v0.31.0 - golang.org/x/term v0.30.0 + golang.org/x/crypto v0.45.0 + golang.org/x/sync v0.18.0 + golang.org/x/sys v0.38.0 + golang.org/x/term v0.37.0 golang.org/x/time v0.9.0 gopkg.in/yaml.v2 v2.4.0 resenje.org/feed v0.1.2 @@ -58,6 +58,8 @@ require ( resenje.org/web v0.4.3 ) +require golang.org/x/telemetry v0.0.0-20251008203120-078029d740a8 // indirect + require ( github.com/BurntSushi/toml v1.1.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect @@ -166,10 +168,10 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/exp v0.0.0-20240213143201-ec583247a57a // indirect - golang.org/x/mod v0.22.0 // indirect - golang.org/x/net v0.38.0 // indirect - golang.org/x/text v0.23.0 // indirect - golang.org/x/tools v0.29.0 // indirect + golang.org/x/mod v0.29.0 // indirect + golang.org/x/net v0.47.0 // indirect + golang.org/x/text v0.31.0 // indirect + golang.org/x/tools v0.38.0 // indirect google.golang.org/protobuf v1.36.1 // indirect gopkg.in/ini.v1 v1.57.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 4badb0a9ca9..3541cb2be1c 100644 --- a/go.sum +++ b/go.sum @@ -1043,8 +1043,8 @@ golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWP golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= -golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= +golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1083,8 +1083,8 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= -golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= +golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1137,8 +1137,8 @@ golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1161,8 +1161,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= -golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1244,13 +1244,15 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/telemetry v0.0.0-20251008203120-078029d740a8 h1:LvzTn0GQhWuvKH/kVRS3R3bVAsdQWI7hvfLHGgh9+lU= +golang.org/x/telemetry v0.0.0-20251008203120-078029d740a8/go.mod h1:Pi4ztBfryZoJEkyFTI5/Ocsu2jXyDr6iSdgJiYE/uwE= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= -golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= +golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= +golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1259,8 +1261,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1321,8 +1323,8 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.29.0 h1:Xx0h3TtM9rzQpQuR4dKLrdglAmCEN5Oi+P74JdhdzXE= -golang.org/x/tools v0.29.0/go.mod h1:KMQVMRsVxU6nHCFXrBPhDB8XncLNLM0lIy/F14RP588= +golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= +golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From a54d62babe2cbce86cd956743eb78f5b2d619ff5 Mon Sep 17 00:00:00 2001 From: significance Date: Fri, 12 Dec 2025 13:27:11 +0000 Subject: [PATCH 59/96] fix: typo (#5299) --- pkg/node/node.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/node/node.go b/pkg/node/node.go index 0a4d526803c..8499bff301a 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -1146,9 +1146,9 @@ func NewBee( detector.OnStabilized = func(t time.Time, totalCount int) { warmupMeasurement(t, totalCount) - reserveTreshold := reserveCapacity >> 1 + reserveThreshold := reserveCapacity >> 1 isFullySynced := func() bool { - return pullerService.SyncRate() == 0 && saludService.IsHealthy() && localStore.ReserveSize() >= reserveTreshold + return pullerService.SyncRate() == 0 && saludService.IsHealthy() && localStore.ReserveSize() >= reserveThreshold } syncCheckTicker := time.NewTicker(2 * time.Second) @@ -1186,9 +1186,9 @@ func NewBee( redistributionContract := redistribution.New(swarmAddress, overlayEthAddress, logger, transactionService, redistributionContractAddress, abiutil.MustParseABI(chainCfg.RedistributionABI), o.TrxDebugMode) isFullySynced := func() bool { - reserveTreshold := reserveCapacity * 5 / 10 + reserveThreshold := reserveCapacity * 5 / 10 logger.Debug("Sync status check evaluated", "stabilized", detector.IsStabilized()) - return localStore.ReserveSize() >= reserveTreshold && pullerService.SyncRate() == 0 && detector.IsStabilized() + return localStore.ReserveSize() >= reserveThreshold && pullerService.SyncRate() == 0 && detector.IsStabilized() } agent, err = storageincentives.New( From d5a095c31ee186064e4186d57293bb8fd64213bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ljubi=C5=A1a=20Ga=C4=8Devi=C4=87?= <35105035+gacevicljubisa@users.noreply.github.com> Date: Thu, 1 Jan 2026 22:21:01 +0100 Subject: [PATCH 60/96] feat(ci): add PR title validation to prevent invalid merge commits (#5304) --- .github/workflows/go.yml | 2 ++ .github/workflows/pr-title.yml | 23 +++++++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 .github/workflows/pr-title.yml diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 1ec8039073f..6e548e238e2 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -66,6 +66,8 @@ jobs: - name: Commit linting if: github.ref != 'refs/heads/master' uses: wagoid/commitlint-github-action@v5 + with: + configFile: commitlint.config.js - name: GolangCI-Lint uses: golangci/golangci-lint-action@v8 with: diff --git a/.github/workflows/pr-title.yml b/.github/workflows/pr-title.yml new file mode 100644 index 00000000000..7958bfacbc4 --- /dev/null +++ b/.github/workflows/pr-title.yml @@ -0,0 +1,23 @@ +name: PR Title Lint + +on: + pull_request: + types: [opened, synchronize, reopened, edited] + +jobs: + lint-title: + name: Lint PR Title + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: '20' + - name: Install commitlint + run: npm install --save-dev @commitlint/cli @commitlint/config-conventional + - name: PR title linting + env: + PR_TITLE: ${{ github.event.pull_request.title }} + run: echo "$PR_TITLE" | npx commitlint --config ./commitlint.config.js From 0d1b99b4ddb71a52c32eb5f81d6769099a3eef1d Mon Sep 17 00:00:00 2001 From: Calin M Date: Fri, 2 Jan 2026 12:09:08 +0200 Subject: [PATCH 61/96] fix: chain backend should not be used for ultra light nodes (#5310) --- pkg/node/node.go | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/pkg/node/node.go b/pkg/node/node.go index 8499bff301a..d17ec0fe83a 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -489,17 +489,19 @@ func NewBee( b.apiCloser = apiServer } - // Sync the with the given Ethereum backend: - isSynced, _, err := transaction.IsSynced(ctx, chainBackend, maxDelay) - if err != nil { - return nil, fmt.Errorf("is synced: %w", err) - } - if !isSynced { - logger.Info("waiting to sync with the blockchain backend") - - err := transaction.WaitSynced(ctx, logger, chainBackend, maxDelay) + if chainEnabled { + // Sync the with the given Ethereum backend: + isSynced, _, err := transaction.IsSynced(ctx, chainBackend, maxDelay) if err != nil { - return nil, fmt.Errorf("waiting backend sync: %w", err) + return nil, fmt.Errorf("is synced: %w", err) + } + if !isSynced { + logger.Info("waiting to sync with the blockchain backend") + + err := transaction.WaitSynced(ctx, logger, chainBackend, maxDelay) + if err != nil { + return nil, fmt.Errorf("waiting backend sync: %w", err) + } } } From 3e157a04cf672ac3b8b49a3c3959bae3d74ccab0 Mon Sep 17 00:00:00 2001 From: acud <12988138+acud@users.noreply.github.com> Date: Sat, 3 Jan 2026 16:22:37 -0600 Subject: [PATCH 62/96] feat: make both feed versions race together (#5287) --- .github/workflows/beekeeper.yml | 9 ++ openapi/Swarm.yaml | 14 +++- openapi/SwarmCommon.yaml | 15 ++-- pkg/api/api.go | 1 + pkg/api/bzz.go | 140 ++++++++++++++++++++++++++------ pkg/api/bzz_test.go | 17 ++-- pkg/api/feed.go | 20 ++--- pkg/api/feed_test.go | 35 +++++--- pkg/api/subdomain.go | 10 +-- pkg/feeds/getter.go | 15 +++- 10 files changed, 195 insertions(+), 81 deletions(-) diff --git a/.github/workflows/beekeeper.yml b/.github/workflows/beekeeper.yml index 4c69fce7a6f..43ac9b18c37 100644 --- a/.github/workflows/beekeeper.yml +++ b/.github/workflows/beekeeper.yml @@ -158,6 +158,9 @@ jobs: - name: Test manifest id: manifest run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-manifest + - name: Test manifest v1 + id: manifest-v1 + run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-manifest-v1 - name: Test postage stamps id: postage-stamps run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks ci-postage @@ -173,6 +176,9 @@ jobs: - name: Test act id: act run: timeout ${TIMEOUT} bash -c 'until beekeeper check --cluster-name local-dns --checks ci-act; do echo "waiting for act..."; sleep .3; done' + - name: Test feeds v1 + id: feeds-v1 + run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-feed-v1 - name: Test feeds id: feeds run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-feed @@ -190,6 +196,9 @@ jobs: if ${{ steps.pushsync-chunks-2.outcome=='failure' }}; then FAILED=pushsync-chunks-2; fi if ${{ steps.retrieval.outcome=='failure' }}; then FAILED=retrieval; fi if ${{ steps.manifest.outcome=='failure' }}; then FAILED=manifest; fi + if ${{ steps.manifest-v1.outcome=='failure' }}; then FAILED=manifest-v1; fi + if ${{ steps.feeds.outcome=='failure' }}; then FAILED=feeds; fi + if ${{ steps.feeds-v1.outcome=='failure' }}; then FAILED=feeds-v1; fi if ${{ steps.act.outcome=='failure' }}; then FAILED=act; fi curl -sSf -X POST -H "Content-Type: application/json" -d "{\"text\": \"**${RUN_TYPE}** Beekeeper Error\nBranch: \`${{ github.head_ref }}\`\nUser: @${{ github.event.pull_request.user.login }}\nDebugging artifacts: [click](https://$BUCKET_NAME.$AWS_ENDPOINT/artifacts_$VERTAG.tar.gz)\nStep failed: \`${FAILED}\`\"}" https://beehive.ethswarm.org/hooks/${{ secrets.TUNSHELL_KEY }} echo "Failed test: ${FAILED}" diff --git a/openapi/Swarm.yaml b/openapi/Swarm.yaml index 7f48e32d329..ce68c014a85 100644 --- a/openapi/Swarm.yaml +++ b/openapi/Swarm.yaml @@ -435,7 +435,6 @@ paths: $ref: "SwarmCommon.yaml#/components/schemas/SwarmReference" required: true description: Swarm address of content - - $ref: "SwarmCommon.yaml#/components/parameters/SwarmFeedLegacyResolve" - $ref: "SwarmCommon.yaml#/components/parameters/SwarmCache" - $ref: "SwarmCommon.yaml#/components/parameters/SwarmRedundancyStrategyParameter" - $ref: "SwarmCommon.yaml#/components/parameters/SwarmRedundancyFallbackModeParameter" @@ -454,6 +453,11 @@ paths: schema: type: string format: binary + headers: + "swarm-feed-resolved-version": + $ref: "SwarmCommon.yaml#/components/headers/SwarmFeedResolvedVersion" + + "400": $ref: "SwarmCommon.yaml#/components/responses/400" "404": @@ -504,7 +508,6 @@ paths: type: string required: true description: Path to the file in the collection. - - $ref: "SwarmCommon.yaml#/components/parameters/SwarmFeedLegacyResolve" - $ref: "SwarmCommon.yaml#/components/parameters/SwarmRedundancyStrategyParameter" - $ref: "SwarmCommon.yaml#/components/parameters/SwarmRedundancyFallbackModeParameter" - $ref: "SwarmCommon.yaml#/components/parameters/SwarmChunkRetrievalTimeoutParameter" @@ -516,6 +519,9 @@ paths: schema: type: string format: binary + headers: + "swarm-feed-resolved-version": + $ref: "SwarmCommon.yaml#/components/headers/SwarmFeedResolvedVersion" "400": $ref: "SwarmCommon.yaml#/components/responses/400" @@ -1032,7 +1038,6 @@ paths: $ref: "SwarmCommon.yaml#/components/schemas/FeedType" required: false description: "Feed indexing scheme (default: sequence)" - - $ref: "SwarmCommon.yaml#/components/parameters/SwarmFeedLegacyResolve" - $ref: "SwarmCommon.yaml#/components/parameters/SwarmOnlyRootChunkParameter" - $ref: "SwarmCommon.yaml#/components/parameters/SwarmCache" - $ref: "SwarmCommon.yaml#/components/parameters/SwarmRedundancyStrategyParameter" @@ -1048,6 +1053,9 @@ paths: $ref: "SwarmCommon.yaml#/components/headers/SwarmFeedIndex" "swarm-feed-index-next": $ref: "SwarmCommon.yaml#/components/headers/SwarmFeedIndexNext" + "swarm-feed-resolved-version": + $ref: "SwarmCommon.yaml#/components/headers/SwarmFeedResolvedVersion" + content: application/octet-stream: schema: diff --git a/openapi/SwarmCommon.yaml b/openapi/SwarmCommon.yaml index bf15dbf205e..1c96d34bf7d 100644 --- a/openapi/SwarmCommon.yaml +++ b/openapi/SwarmCommon.yaml @@ -1071,6 +1071,13 @@ components: schema: type: string + SwarmFeedResolvedVersion: + schema: + type: string + required: false + description: "Indicates which feed version was resolved (v1 or v2)" + + parameters: GasPriceParameter: in: header @@ -1276,14 +1283,6 @@ components: required: false description: "ACT history Unix timestamp" - SwarmFeedLegacyResolve: - in: query - name: swarm-feed-legacy-resolve - schema: - type: boolean - required: false - description: "Resolves feed payloads in legacy structure (timestamp, content address)." - responses: "200": description: OK. diff --git a/pkg/api/api.go b/pkg/api/api.go index 5f747237bc4..3ab9a0eabb4 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -79,6 +79,7 @@ const ( SwarmSocSignatureHeader = "Swarm-Soc-Signature" SwarmFeedIndexHeader = "Swarm-Feed-Index" SwarmFeedIndexNextHeader = "Swarm-Feed-Index-Next" + SwarmFeedResolvedVersionHeader = "Swarm-Feed-Resolved-Version" SwarmOnlyRootChunk = "Swarm-Only-Root-Chunk" SwarmCollectionHeader = "Swarm-Collection" SwarmPostageBatchIdHeader = "Swarm-Postage-Batch-Id" diff --git a/pkg/api/bzz.go b/pkg/api/bzz.go index a27e3f813c5..e547f6f951d 100644 --- a/pkg/api/bzz.go +++ b/pkg/api/bzz.go @@ -337,15 +337,7 @@ func (s *Service) bzzDownloadHandler(w http.ResponseWriter, r *http.Request) { paths.Path = strings.TrimRight(paths.Path, "/") + "/" // NOTE: leave one slash if there was some. } - queries := struct { - FeedLegacyResolve bool `map:"swarm-feed-legacy-resolve"` - }{} - if response := s.mapStructure(r.URL.Query(), &queries); response != nil { - response("invalid query params", logger, w) - return - } - - s.serveReference(logger, address, paths.Path, w, r, false, queries.FeedLegacyResolve) + s.serveReference(logger, address, paths.Path, w, r, false) } func (s *Service) bzzHeadHandler(w http.ResponseWriter, r *http.Request) { @@ -360,14 +352,6 @@ func (s *Service) bzzHeadHandler(w http.ResponseWriter, r *http.Request) { return } - queries := struct { - FeedLegacyResolve bool `map:"swarm-feed-legacy-resolve"` - }{} - if response := s.mapStructure(r.URL.Query(), &queries); response != nil { - response("invalid query params", logger, w) - return - } - address := paths.Address if v := getAddressFromContext(r.Context()); !v.IsZero() { address = v @@ -377,10 +361,119 @@ func (s *Service) bzzHeadHandler(w http.ResponseWriter, r *http.Request) { paths.Path = strings.TrimRight(paths.Path, "/") + "/" // NOTE: leave one slash if there was some. } - s.serveReference(logger, address, paths.Path, w, r, true, queries.FeedLegacyResolve) + s.serveReference(logger, address, paths.Path, w, r, true) } -func (s *Service) serveReference(logger log.Logger, address swarm.Address, pathVar string, w http.ResponseWriter, r *http.Request, headerOnly bool, feedLegacyResolve bool) { +type getWrappedResult struct { + ch swarm.Chunk + v1 bool // indicates whether the feed that was resolved is v1. false if v2 + err error +} + +// resolveFeed races the resolution of both types of feeds. +// figure out if its a v1 or v2 chunk. +// it returns the first correct feed found, the type found ("v1" or "v2") or an error. +func (s *Service) resolveFeed(ctx context.Context, getter storage.Getter, ch swarm.Chunk) (swarm.Chunk, string, error) { + innerCtx, cancel := context.WithCancel(ctx) + defer cancel() + getWrapped := func(v1 bool) chan getWrappedResult { + ret := make(chan getWrappedResult) + go func() { + wc, err := feeds.GetWrappedChunk(innerCtx, getter, ch, v1) + if err != nil { + select { + case ret <- getWrappedResult{nil, v1, err}: + return + case <-innerCtx.Done(): + return + } + } + + // here we just check whether the address is retrievable. + // if it returns an error we send that over the channel, otherwise + // we send the wc chunk back to the caller so that the feed can be + // dereferenced. + _, err = getter.Get(innerCtx, wc.Address()) + if err != nil { + select { + case ret <- getWrappedResult{wc, v1, err}: + return + case <-innerCtx.Done(): + return + } + } + select { + case ret <- getWrappedResult{wc, v1, nil}: + return + case <-innerCtx.Done(): + return + } + }() + return ret + } + isV1, err := feeds.IsV1Payload(ch) + if err != nil { + return nil, "", err + } + // if we have v1 length, it means there's ambiguity so we + // should fetch both feed versions. if the length isn't v1 + // then we should only try to fetch v2. + var ( + v1C, v2C chan getWrappedResult + both = false + ) + if isV1 { + both = true + v1C = getWrapped(true) + v2C = getWrapped(false) + } else { + v2C = getWrapped(false) + } + + // closure to handle processing one channel then the other. + // the "resolving" parameter is meant to tell the closure which feed type is in the result struct + // which in turns allows it to return which feed type was resolved. + processChanOutput := func(resolving string, result getWrappedResult, other chan getWrappedResult) (swarm.Chunk, string, error) { + defer cancel() + if !both { + if resolving == "v2" { + return result.ch, resolving, nil + } + return result.ch, resolving, result.err + } + // both are being checked. if there's no err return the chunk + // otherwise wait for the other channel + if result.err == nil { + return result.ch, resolving, nil + } + if resolving == "v1" { + resolving = "v2" + } else { + resolving = "v1" + } + // wait for the other one + select { + case result := <-other: + if !result.v1 { + // resolving v2 + return result.ch, resolving, nil + } + return result.ch, resolving, result.err + case <-innerCtx.Done(): + return nil, "", ctx.Err() + } + } + select { + case v1r := <-v1C: + return processChanOutput("v1", v1r, v2C) + case v2r := <-v2C: + return processChanOutput("v2", v2r, v1C) + case <-innerCtx.Done(): + return nil, "", ctx.Err() + } +} + +func (s *Service) serveReference(logger log.Logger, address swarm.Address, pathVar string, w http.ResponseWriter, r *http.Request, headerOnly bool) { loggerV1 := logger.V(1).Build() headers := struct { @@ -415,7 +508,6 @@ func (s *Service) serveReference(logger log.Logger, address swarm.Address, pathV jsonhttp.BadRequest(w, "could not parse headers") return } - FETCH: // read manifest entry m, err := manifest.NewDefaultManifestReference( @@ -449,7 +541,8 @@ FETCH: jsonhttp.NotFound(w, "no update found") return } - wc, err := feeds.GetWrappedChunk(ctx, s.storer.Download(cache), ch, feedLegacyResolve) + + wc, feedVer, err := s.resolveFeed(ctx, s.storer.Download(cache), ch) if err != nil { if errors.Is(err, feeds.ErrNotLegacyPayload) { logger.Debug("bzz: download: feed is not a legacy payload") @@ -468,10 +561,10 @@ FETCH: jsonhttp.InternalServerError(w, "mapStructure feed update") return } + address = wc.Address() // modify ls and init with non-existing wrapped chunk ls = loadsave.NewReadonlyWithRootCh(s.storer.Download(cache), s.storer.Cache(), wc, rLevel) - feedDereferenced = true curBytes, err := cur.MarshalBinary() if err != nil { @@ -482,6 +575,7 @@ FETCH: } w.Header().Set(SwarmFeedIndexHeader, hex.EncodeToString(curBytes)) + w.Header().Set(SwarmFeedResolvedVersionHeader, feedVer) // this header might be overriding others. handle with care. in the future // we should implement an append functionality for this specific header, // since different parts of handlers might be overriding others' values @@ -490,7 +584,6 @@ FETCH: goto FETCH } } - if pathVar == "" { loggerV1.Debug("bzz download: handle empty path", "address", address) @@ -505,6 +598,7 @@ FETCH: return } } + logger.Debug("bzz download: address not found or incorrect", "address", address, "path", pathVar) logger.Error(nil, "address not found or incorrect") jsonhttp.NotFound(w, "address not found or incorrect") diff --git a/pkg/api/bzz_test.go b/pkg/api/bzz_test.go index 7f636476d1f..c12c108d418 100644 --- a/pkg/api/bzz_test.go +++ b/pkg/api/bzz_test.go @@ -764,12 +764,8 @@ func TestFeedIndirection(t *testing.T) { Logger: logger, Post: mockpost.New(mockpost.WithAcceptAll()), }) - bzzDownloadResource = func(addr, path string, legacyFeed bool) string { + bzzDownloadResource = func(addr, path string) string { values := url.Values{} - if legacyFeed { - values.Set("swarm-feed-legacy-resolve", strconv.FormatBool(legacyFeed)) - } - baseURL := "/bzz/" + addr + "/" + path if len(values) > 0 { return baseURL + "?" + values.Encode() @@ -835,7 +831,6 @@ func TestFeedIndirection(t *testing.T) { t.Run("legacy feed", func(t *testing.T) { feedUpdate := toChunk(t, 121212, resp.Reference.Bytes()) - var ( look = newMockLookup(-1, 0, feedUpdate, nil, &id{}, nil) factory = newMockFactory(look) @@ -846,16 +841,15 @@ func TestFeedIndirection(t *testing.T) { Feeds: factory, }) - jsonhttptest.Request(t, client, http.MethodGet, bzzDownloadResource(manifRef.String(), "", true), http.StatusOK, + jsonhttptest.Request(t, client, http.MethodGet, bzzDownloadResource(manifRef.String(), ""), http.StatusOK, jsonhttptest.WithExpectedResponse(updateData), jsonhttptest.WithExpectedContentLength(len(updateData)), jsonhttptest.WithExpectedResponseHeader(api.AccessControlExposeHeaders, api.SwarmFeedIndexHeader), jsonhttptest.WithExpectedResponseHeader(api.AccessControlExposeHeaders, api.ContentDispositionHeader), jsonhttptest.WithExpectedResponseHeader(api.ContentDispositionHeader, `inline; filename="index.html"`), jsonhttptest.WithExpectedResponseHeader(api.ContentTypeHeader, "text/html; charset=utf-8"), + jsonhttptest.WithExpectedResponseHeader(api.SwarmFeedResolvedVersionHeader, "v1"), ) - - jsonhttptest.Request(t, client, http.MethodGet, bzzDownloadResource(manifRef.String(), "", false), http.StatusNotFound) }) t.Run("wrapped feed", func(t *testing.T) { @@ -876,16 +870,15 @@ func TestFeedIndirection(t *testing.T) { Feeds: factory, }) - jsonhttptest.Request(t, client, http.MethodGet, bzzDownloadResource(manifRef.String(), "", false), http.StatusOK, + jsonhttptest.Request(t, client, http.MethodGet, bzzDownloadResource(manifRef.String(), ""), http.StatusOK, jsonhttptest.WithExpectedResponse(updateData), jsonhttptest.WithExpectedContentLength(len(updateData)), jsonhttptest.WithExpectedResponseHeader(api.AccessControlExposeHeaders, api.SwarmFeedIndexHeader), jsonhttptest.WithExpectedResponseHeader(api.AccessControlExposeHeaders, api.ContentDispositionHeader), jsonhttptest.WithExpectedResponseHeader(api.ContentDispositionHeader, `inline; filename="index.html"`), jsonhttptest.WithExpectedResponseHeader(api.ContentTypeHeader, "text/html; charset=utf-8"), + jsonhttptest.WithExpectedResponseHeader(api.SwarmFeedResolvedVersionHeader, "v2"), ) - - jsonhttptest.Request(t, client, http.MethodGet, bzzDownloadResource(manifRef.String(), "", true), http.StatusBadRequest) }) } diff --git a/pkg/api/feed.go b/pkg/api/feed.go index 313b04c831a..cc73fc6f3df 100644 --- a/pkg/api/feed.go +++ b/pkg/api/feed.go @@ -53,9 +53,8 @@ func (s *Service) feedGetHandler(w http.ResponseWriter, r *http.Request) { } queries := struct { - At int64 `map:"at"` - After uint64 `map:"after"` - FeedLegacyResolve bool `map:"swarm-feed-legacy-resolve"` + At int64 `map:"at"` + After uint64 `map:"after"` }{} if response := s.mapStructure(r.URL.Query(), &queries); response != nil { response("invalid query params", logger, w) @@ -72,7 +71,6 @@ func (s *Service) feedGetHandler(w http.ResponseWriter, r *http.Request) { response("invalid header params", logger, w) return } - f := feeds.New(paths.Topic, paths.Owner) lookup, err := s.feedFactory.NewLookup(feeds.Sequence, f) if err != nil { @@ -86,7 +84,6 @@ func (s *Service) feedGetHandler(w http.ResponseWriter, r *http.Request) { } return } - ch, cur, next, err := lookup.At(r.Context(), queries.At, queries.After) if err != nil { logger.Debug("lookup at failed", "at", queries.At, "error", err) @@ -103,7 +100,7 @@ func (s *Service) feedGetHandler(w http.ResponseWriter, r *http.Request) { return } - wc, err := feeds.GetWrappedChunk(r.Context(), s.storer.Download(false), ch, queries.FeedLegacyResolve) + wc, feedVer, err := s.resolveFeed(r.Context(), s.storer.Download(false), ch) if err != nil { logger.Error(nil, "wrapped chunk cannot be retrieved") jsonhttp.NotFound(w, "wrapped chunk cannot be retrieved") @@ -135,11 +132,12 @@ func (s *Service) feedGetHandler(w http.ResponseWriter, r *http.Request) { sig := socCh.Signature() additionalHeaders := http.Header{ - ContentTypeHeader: {"application/octet-stream"}, - SwarmFeedIndexHeader: {hex.EncodeToString(curBytes)}, - SwarmFeedIndexNextHeader: {hex.EncodeToString(nextBytes)}, - SwarmSocSignatureHeader: {hex.EncodeToString(sig)}, - AccessControlExposeHeaders: {SwarmFeedIndexHeader, SwarmFeedIndexNextHeader, SwarmSocSignatureHeader}, + ContentTypeHeader: {"application/octet-stream"}, + SwarmFeedIndexHeader: {hex.EncodeToString(curBytes)}, + SwarmFeedIndexNextHeader: {hex.EncodeToString(nextBytes)}, + SwarmSocSignatureHeader: {hex.EncodeToString(sig)}, + SwarmFeedResolvedVersionHeader: {feedVer}, + AccessControlExposeHeaders: {SwarmFeedIndexHeader, SwarmFeedIndexNextHeader, SwarmSocSignatureHeader}, } if headers.OnlyRootChunk { diff --git a/pkg/api/feed_test.go b/pkg/api/feed_test.go index fcfe4b93e68..e33eba63d0c 100644 --- a/pkg/api/feed_test.go +++ b/pkg/api/feed_test.go @@ -15,7 +15,6 @@ import ( "math/big" "net/http" "net/url" - "strconv" "testing" "github.com/ethersphere/bee/v2/pkg/api" @@ -44,14 +43,11 @@ func TestFeed_Get(t *testing.T) { t.Parallel() var ( - feedResource = func(owner, topic, at string, legacyFeed bool) string { + feedResource = func(owner, topic, at string) string { values := url.Values{} if at != "" { values.Set("at", at) } - if legacyFeed { - values.Set("swarm-feed-legacy-resolve", strconv.FormatBool(legacyFeed)) - } baseURL := fmt.Sprintf("/feeds/%s/%s", owner, topic) if len(values) > 0 { @@ -86,7 +82,7 @@ func TestFeed_Get(t *testing.T) { }) ) - jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", "12", true), http.StatusOK, + jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", "12"), http.StatusOK, jsonhttptest.WithExpectedResponse(mockWrappedCh.Data()[swarm.SpanSize:]), jsonhttptest.WithExpectedResponseHeader(api.SwarmFeedIndexHeader, hex.EncodeToString(idBytes)), jsonhttptest.WithExpectedResponseHeader(api.AccessControlExposeHeaders, api.SwarmFeedIndexHeader), @@ -94,6 +90,7 @@ func TestFeed_Get(t *testing.T) { jsonhttptest.WithExpectedResponseHeader(api.AccessControlExposeHeaders, api.SwarmSocSignatureHeader), jsonhttptest.WithExpectedResponseHeader(api.AccessControlExposeHeaders, api.ContentDispositionHeader), jsonhttptest.WithExpectedResponseHeader(api.ContentTypeHeader, "application/octet-stream"), + jsonhttptest.WithExpectedResponseHeader(api.SwarmFeedResolvedVersionHeader, "v1"), ) }) @@ -113,7 +110,7 @@ func TestFeed_Get(t *testing.T) { }) ) - jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", "", true), http.StatusOK, + jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", ""), http.StatusOK, jsonhttptest.WithExpectedResponse(mockWrappedCh.Data()[swarm.SpanSize:]), jsonhttptest.WithExpectedContentLength(len(mockWrappedCh.Data()[swarm.SpanSize:])), jsonhttptest.WithExpectedResponseHeader(api.SwarmFeedIndexHeader, hex.EncodeToString(idBytes)), @@ -122,6 +119,7 @@ func TestFeed_Get(t *testing.T) { jsonhttptest.WithExpectedResponseHeader(api.AccessControlExposeHeaders, api.SwarmSocSignatureHeader), jsonhttptest.WithExpectedResponseHeader(api.AccessControlExposeHeaders, api.ContentDispositionHeader), jsonhttptest.WithExpectedResponseHeader(api.ContentTypeHeader, "application/octet-stream"), + jsonhttptest.WithExpectedResponseHeader(api.SwarmFeedResolvedVersionHeader, "v1"), ) }) @@ -142,7 +140,7 @@ func TestFeed_Get(t *testing.T) { }) ) - jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", "", false), http.StatusOK, + jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", ""), http.StatusOK, jsonhttptest.WithExpectedResponse(testData), jsonhttptest.WithExpectedContentLength(len(testData)), jsonhttptest.WithExpectedResponseHeader(api.SwarmFeedIndexHeader, hex.EncodeToString(idBytes)), @@ -151,11 +149,18 @@ func TestFeed_Get(t *testing.T) { jsonhttptest.WithExpectedResponseHeader(api.AccessControlExposeHeaders, api.SwarmSocSignatureHeader), jsonhttptest.WithExpectedResponseHeader(api.AccessControlExposeHeaders, api.ContentDispositionHeader), jsonhttptest.WithExpectedResponseHeader(api.ContentTypeHeader, "application/octet-stream"), + jsonhttptest.WithExpectedResponseHeader(api.SwarmFeedResolvedVersionHeader, "v2"), ) }) t.Run("legacy payload with non existing wrapped chunk", func(t *testing.T) { - t.Parallel() + t.Skip() + /* + This test has been disabled since it cannot be supported with the automatic + Feed resolution logic that is now in place. In case automatic feed resolution + would be removed at some point, this test can be reactived. The issue is + thoroughly described in the PR: https://github.com/ethersphere/bee/pull/5287 + */ wrappedRef := make([]byte, swarm.HashSize) _ = copy(wrappedRef, mockWrappedCh.Address().Bytes()) @@ -172,7 +177,7 @@ func TestFeed_Get(t *testing.T) { }) ) - jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", "", true), http.StatusNotFound) + jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", ""), http.StatusNotFound) }) t.Run("query parameter legacy feed resolve", func(t *testing.T) { @@ -188,7 +193,7 @@ func TestFeed_Get(t *testing.T) { ) // Test with the legacyFeed parameter set to true which should add the query parameter - jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", "", false), http.StatusNotFound) + jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", ""), http.StatusNotFound) }) t.Run("bigger payload than one chunk", func(t *testing.T) { @@ -220,7 +225,7 @@ func TestFeed_Get(t *testing.T) { ) t.Run("retrieve chunk tree", func(t *testing.T) { - jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", "", false), http.StatusOK, + jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", ""), http.StatusOK, jsonhttptest.WithExpectedResponse(testData), jsonhttptest.WithExpectedContentLength(testDataLen), jsonhttptest.WithExpectedResponseHeader(api.SwarmFeedIndexHeader, hex.EncodeToString(idBytes)), @@ -229,11 +234,12 @@ func TestFeed_Get(t *testing.T) { jsonhttptest.WithExpectedResponseHeader(api.AccessControlExposeHeaders, api.SwarmSocSignatureHeader), jsonhttptest.WithExpectedResponseHeader(api.AccessControlExposeHeaders, api.ContentDispositionHeader), jsonhttptest.WithExpectedResponseHeader(api.ContentTypeHeader, "application/octet-stream"), + jsonhttptest.WithExpectedResponseHeader(api.SwarmFeedResolvedVersionHeader, "v2"), ) }) t.Run("retrieve only wrapped chunk", func(t *testing.T) { - jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", "", false), http.StatusOK, + jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", ""), http.StatusOK, jsonhttptest.WithRequestHeader(api.SwarmOnlyRootChunk, "true"), jsonhttptest.WithExpectedResponse(testRootCh.Data()), jsonhttptest.WithExpectedContentLength(len(testRootCh.Data())), @@ -242,6 +248,7 @@ func TestFeed_Get(t *testing.T) { jsonhttptest.WithExpectedResponseHeader(api.AccessControlExposeHeaders, api.SwarmFeedIndexNextHeader), jsonhttptest.WithExpectedResponseHeader(api.AccessControlExposeHeaders, api.SwarmSocSignatureHeader), jsonhttptest.WithExpectedResponseHeader(api.ContentTypeHeader, "application/octet-stream"), + jsonhttptest.WithExpectedResponseHeader(api.SwarmFeedResolvedVersionHeader, "v2"), ) }) }) @@ -384,9 +391,11 @@ func (l *mockLookup) At(_ context.Context, at int64, after uint64) (swarm.Chunk, // shortcut to ignore the value in the call since time.Now() is a moving target return l.chunk, l.cur, l.next, nil } + if at == l.at && after == l.after { return l.chunk, l.cur, l.next, nil } + return nil, nil, nil, errors.New("no feed update found") } diff --git a/pkg/api/subdomain.go b/pkg/api/subdomain.go index b30e270a4ec..5d5cede467a 100644 --- a/pkg/api/subdomain.go +++ b/pkg/api/subdomain.go @@ -28,13 +28,5 @@ func (s *Service) subdomainHandler(w http.ResponseWriter, r *http.Request) { paths.Path = strings.TrimRight(paths.Path, "/") + "/" // NOTE: leave one slash if there was some. } - queries := struct { - FeedLegacyResolve bool `map:"swarm-feed-legacy-resolve"` - }{} - if response := s.mapStructure(r.URL.Query(), &queries); response != nil { - response("invalid query params", logger, w) - return - } - - s.serveReference(logger, paths.Subdomain, paths.Path, w, r, false, queries.FeedLegacyResolve) + s.serveReference(logger, paths.Subdomain, paths.Path, w, r, false) } diff --git a/pkg/feeds/getter.go b/pkg/feeds/getter.go index 2bdf509e282..40ccf98bca7 100644 --- a/pkg/feeds/getter.go +++ b/pkg/feeds/getter.go @@ -93,9 +93,20 @@ func FromChunk(ch swarm.Chunk) (swarm.Chunk, error) { // legacyPayload returns back the referenced chunk and datetime from the legacy feed payload func legacyPayload(wrappedChunk swarm.Chunk) (swarm.Address, error) { cacData := wrappedChunk.Data() - if len(cacData) != 16+swarm.HashSize && len(cacData) != 16+swarm.HashSize*2 { + if !isV1Length(len(cacData)) { return swarm.ZeroAddress, ErrNotLegacyPayload } - return swarm.NewAddress(cacData[16:]), nil } + +func IsV1Payload(ch swarm.Chunk) (bool, error) { + cc, err := FromChunk(ch) + if err != nil { + return false, err + } + return isV1Length(len(cc.Data())), nil +} + +func isV1Length(length int) bool { + return length == 16+swarm.HashSize || length == 16+swarm.HashSize*2 +} From 47cb8d1db80a9b3e09dd746070d422e6bea59404 Mon Sep 17 00:00:00 2001 From: Akrem Chabchoub <121046693+akrem-chabchoub@users.noreply.github.com> Date: Thu, 8 Jan 2026 11:13:58 +0100 Subject: [PATCH 63/96] test(pingpong): add synctest to ping test (#5264) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Janoš Guljaš --- pkg/p2p/streamtest/streamtest.go | 19 +++- pkg/p2p/streamtest/streamtest_test.go | 45 ++++++++ pkg/pingpong/pingpong_test.go | 151 ++++++++++++-------------- 3 files changed, 133 insertions(+), 82 deletions(-) diff --git a/pkg/p2p/streamtest/streamtest.go b/pkg/p2p/streamtest/streamtest.go index 92e538d83bd..3265f6ea566 100644 --- a/pkg/p2p/streamtest/streamtest.go +++ b/pkg/p2p/streamtest/streamtest.go @@ -38,6 +38,7 @@ type Recorder struct { streamErr func(swarm.Address, string, string, string) error pingErr func(ma.Multiaddr) (time.Duration, error) protocolsWithPeers map[string]p2p.ProtocolSpec + messageLatency time.Duration } func WithProtocols(protocols ...p2p.ProtocolSpec) Option { @@ -82,6 +83,12 @@ func WithPingErr(pingErr func(ma.Multiaddr) (time.Duration, error)) Option { }) } +func WithMessageLatency(latency time.Duration) Option { + return optionFunc(func(r *Recorder) { + r.messageLatency = latency + }) +} + func New(opts ...Option) *Recorder { r := &Recorder{ records: make(map[string][]*Record), @@ -115,8 +122,8 @@ func (r *Recorder) NewStream(ctx context.Context, addr swarm.Address, h p2p.Head } } - recordIn := newRecord() - recordOut := newRecord() + recordIn := newRecord(r.messageLatency) + recordOut := newRecord(r.messageLatency) streamOut := newStream(recordIn, recordOut) streamIn := newStream(recordOut, recordIn) @@ -328,16 +335,20 @@ type record struct { c int lock sync.Mutex dataSigC chan struct{} + latency time.Duration closed bool } -func newRecord() *record { +func newRecord(latency time.Duration) *record { return &record{ dataSigC: make(chan struct{}, 16), + latency: latency, } } func (r *record) Read(p []byte) (n int, err error) { + defer time.Sleep(r.latency) + for r.c == r.bytesSize() { _, ok := <-r.dataSigC if !ok { @@ -356,6 +367,8 @@ func (r *record) Read(p []byte) (n int, err error) { } func (r *record) Write(p []byte) (int, error) { + defer time.Sleep(r.latency) + r.lock.Lock() defer r.lock.Unlock() diff --git a/pkg/p2p/streamtest/streamtest_test.go b/pkg/p2p/streamtest/streamtest_test.go index 7dd1ae02091..51b0d903ef2 100644 --- a/pkg/p2p/streamtest/streamtest_test.go +++ b/pkg/p2p/streamtest/streamtest_test.go @@ -12,6 +12,7 @@ import ( "io" "strings" "testing" + "testing/synctest" "time" "github.com/ethersphere/bee/v2/pkg/p2p" @@ -785,6 +786,50 @@ func TestRecorder_ping(t *testing.T) { } } +func TestRecorder_WithMessageLatency(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + latency := 200 * time.Millisecond + recorder := streamtest.New( + streamtest.WithMessageLatency(latency), + streamtest.WithProtocols( + newTestProtocol(func(_ context.Context, peer p2p.Peer, stream p2p.Stream) error { + rw := bufio.NewReadWriter(bufio.NewReader(stream), bufio.NewWriter(stream)) + if _, err := rw.ReadString('\n'); err != nil { + return err + } + if _, err := rw.WriteString("pong\n"); err != nil { + return err + } + return rw.Flush() + }), + ), + ) + + stream, err := recorder.NewStream(context.Background(), swarm.ZeroAddress, nil, testProtocolName, testProtocolVersion, testStreamName) + if err != nil { + t.Fatal(err) + } + defer stream.Close() + + start := time.Now() + if _, err := stream.Write([]byte("ping\n")); err != nil { + t.Fatal(err) + } + if duration := time.Since(start); duration < latency { + t.Errorf("write took %v, want >= %v", duration, latency) + } + + start = time.Now() + rw := bufio.NewReader(stream) + if _, err := rw.ReadString('\n'); err != nil { + t.Fatal(err) + } + if duration := time.Since(start); duration < latency { + t.Errorf("read took %v, want >= %v", duration, latency) + } + }) +} + const ( testProtocolName = "testing" testProtocolVersion = "1.0.1" diff --git a/pkg/pingpong/pingpong_test.go b/pkg/pingpong/pingpong_test.go index 6e0298b7cb9..99c71a6dce6 100644 --- a/pkg/pingpong/pingpong_test.go +++ b/pkg/pingpong/pingpong_test.go @@ -8,14 +8,13 @@ import ( "bytes" "context" "fmt" - "runtime" "testing" + "testing/synctest" "time" "github.com/ethersphere/bee/v2/pkg/swarm" "github.com/ethersphere/bee/v2/pkg/log" - "github.com/ethersphere/bee/v2/pkg/p2p" "github.com/ethersphere/bee/v2/pkg/p2p/protobuf" "github.com/ethersphere/bee/v2/pkg/p2p/streamtest" "github.com/ethersphere/bee/v2/pkg/pingpong" @@ -23,91 +22,85 @@ import ( ) func TestPing(t *testing.T) { - t.Parallel() + synctest.Test(t, func(t *testing.T) { + logger := log.Noop - logger := log.Noop + // create a pingpong server that handles the incoming stream + server := pingpong.New(nil, logger, nil) - // create a pingpong server that handles the incoming stream - server := pingpong.New(nil, logger, nil) + messageLatency := 200 * time.Millisecond - // setup the stream recorder to record stream data - recorder := streamtest.New( - streamtest.WithProtocols(server.Protocol()), - streamtest.WithMiddlewares(func(f p2p.HandlerFunc) p2p.HandlerFunc { - if runtime.GOOS == "windows" { - // windows has a bit lower time resolution - // so, slow down the handler with a middleware - // not to get 0s for rtt value - time.Sleep(100 * time.Millisecond) - } - return f - }), - ) + // setup the stream recorder to record stream data + recorder := streamtest.New( + streamtest.WithProtocols(server.Protocol()), + streamtest.WithMessageLatency(messageLatency), + ) - // create a pingpong client that will do pinging - client := pingpong.New(recorder, logger, nil) + // create a pingpong client that will do pinging + client := pingpong.New(recorder, logger, nil) - // ping - addr := swarm.MustParseHexAddress("ca1e9f3938cc1425c6061b96ad9eb93e134dfe8734ad490164ef20af9d1cf59c") - greetings := []string{"hey", "there", "fella"} - rtt, err := client.Ping(context.Background(), addr, greetings...) - if err != nil { - t.Fatal(err) - } + // ping + addr := swarm.MustParseHexAddress("ca1e9f3938cc1425c6061b96ad9eb93e134dfe8734ad490164ef20af9d1cf59c") + greetings := []string{"hey", "there", "fella"} + rtt, err := client.Ping(context.Background(), addr, greetings...) + if err != nil { + t.Fatal(err) + } - // check that RTT is a sane value - if rtt <= 0 { - t.Errorf("invalid RTT value %v", rtt) - } + // check that RTT is the sum of all message latencies + if rtt != time.Duration(2*len(greetings))*messageLatency { + t.Errorf("invalid RTT value %v", rtt) + } - // get a record for this stream - records, err := recorder.Records(addr, "pingpong", "1.0.0", "pingpong") - if err != nil { - t.Fatal(err) - } - if l := len(records); l != 1 { - t.Fatalf("got %v records, want %v", l, 1) - } - record := records[0] + // get a record for this stream + records, err := recorder.Records(addr, "pingpong", "1.0.0", "pingpong") + if err != nil { + t.Fatal(err) + } + if l := len(records); l != 1 { + t.Fatalf("got %v records, want %v", l, 1) + } + record := records[0] - // validate received ping greetings from the client - wantGreetings := greetings - messages, err := protobuf.ReadMessages( - bytes.NewReader(record.In()), - func() protobuf.Message { return new(pb.Ping) }, - ) - if err != nil { - t.Fatal(err) - } - gotGreetings := make([]string, 0, len(messages)) - for _, m := range messages { - gotGreetings = append(gotGreetings, m.(*pb.Ping).Greeting) - } - if fmt.Sprint(gotGreetings) != fmt.Sprint(wantGreetings) { - t.Errorf("got greetings %v, want %v", gotGreetings, wantGreetings) - } + // validate received ping greetings from the client + wantGreetings := greetings + messages, err := protobuf.ReadMessages( + bytes.NewReader(record.In()), + func() protobuf.Message { return new(pb.Ping) }, + ) + if err != nil { + t.Fatal(err) + } + gotGreetings := make([]string, 0, len(messages)) + for _, m := range messages { + gotGreetings = append(gotGreetings, m.(*pb.Ping).Greeting) + } + if fmt.Sprint(gotGreetings) != fmt.Sprint(wantGreetings) { + t.Errorf("got greetings %v, want %v", gotGreetings, wantGreetings) + } - // validate sent pong responses by handler - wantResponses := make([]string, 0, len(greetings)) - for _, g := range greetings { - wantResponses = append(wantResponses, "{"+g+"}") - } - messages, err = protobuf.ReadMessages( - bytes.NewReader(record.Out()), - func() protobuf.Message { return new(pb.Pong) }, - ) - if err != nil { - t.Fatal(err) - } - gotResponses := make([]string, 0, len(messages)) - for _, m := range messages { - gotResponses = append(gotResponses, m.(*pb.Pong).Response) - } - if fmt.Sprint(gotResponses) != fmt.Sprint(wantResponses) { - t.Errorf("got responses %v, want %v", gotResponses, wantResponses) - } + // validate sent pong responses by handler + wantResponses := make([]string, 0, len(greetings)) + for _, g := range greetings { + wantResponses = append(wantResponses, "{"+g+"}") + } + messages, err = protobuf.ReadMessages( + bytes.NewReader(record.Out()), + func() protobuf.Message { return new(pb.Pong) }, + ) + if err != nil { + t.Fatal(err) + } + gotResponses := make([]string, 0, len(messages)) + for _, m := range messages { + gotResponses = append(gotResponses, m.(*pb.Pong).Response) + } + if fmt.Sprint(gotResponses) != fmt.Sprint(wantResponses) { + t.Errorf("got responses %v, want %v", gotResponses, wantResponses) + } - if err := record.Err(); err != nil { - t.Fatal(err) - } + if err := record.Err(); err != nil { + t.Fatal(err) + } + }) } From f612bf91ec08546569eed299cb0934ec64380d14 Mon Sep 17 00:00:00 2001 From: vicerace Date: Thu, 8 Jan 2026 18:26:44 +0800 Subject: [PATCH 64/96] chore: updated mock accounting service function names and comments (#5293) Signed-off-by: vicerace --- pkg/accounting/mock/accounting.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/accounting/mock/accounting.go b/pkg/accounting/mock/accounting.go index e5e0d98130e..cbf838f9ba3 100644 --- a/pkg/accounting/mock/accounting.go +++ b/pkg/accounting/mock/accounting.go @@ -44,14 +44,14 @@ type creditAction struct { applied bool } -// WithDebitFunc sets the mock Debit function +// WithPrepareDebitFunc sets the mock PrepareDebit function func WithPrepareDebitFunc(f func(peer swarm.Address, price uint64) (accounting.Action, error)) Option { return optionFunc(func(s *Service) { s.prepareDebitFunc = f }) } -// WithDebitFunc sets the mock Debit function +// WithPrepareCreditFunc sets the mock PrepareCredit function func WithPrepareCreditFunc(f func(peer swarm.Address, price uint64, originated bool) (accounting.Action, error)) Option { return optionFunc(func(s *Service) { s.prepareCreditFunc = f From 55d4864ad9b4032b4ed99a5ab4815e49172f0a4e Mon Sep 17 00:00:00 2001 From: Akrem Chabchoub <121046693+akrem-chabchoub@users.noreply.github.com> Date: Thu, 8 Jan 2026 15:04:48 +0100 Subject: [PATCH 65/96] fix(node): allow hostnames in public address validation (#5312) --- go.mod | 2 +- pkg/node/node.go | 26 +++++++++++++++++++------- pkg/node/node_test.go | 20 +++++++++++++++----- 3 files changed, 35 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index a0439b609ed..a55a0923d50 100644 --- a/go.mod +++ b/go.mod @@ -47,6 +47,7 @@ require ( go.uber.org/atomic v1.11.0 go.uber.org/goleak v1.3.0 golang.org/x/crypto v0.45.0 + golang.org/x/net v0.47.0 golang.org/x/sync v0.18.0 golang.org/x/sys v0.38.0 golang.org/x/term v0.37.0 @@ -169,7 +170,6 @@ require ( go.uber.org/zap v1.27.0 // indirect golang.org/x/exp v0.0.0-20240213143201-ec583247a57a // indirect golang.org/x/mod v0.29.0 // indirect - golang.org/x/net v0.47.0 // indirect golang.org/x/text v0.31.0 // indirect golang.org/x/tools v0.38.0 // indirect google.golang.org/protobuf v1.36.1 // indirect diff --git a/pkg/node/node.go b/pkg/node/node.go index d17ec0fe83a..f20468df338 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -81,6 +81,7 @@ import ( ma "github.com/multiformats/go-multiaddr" "github.com/prometheus/client_golang/prometheus" "golang.org/x/crypto/sha3" + "golang.org/x/net/idna" "golang.org/x/sync/errgroup" ) @@ -1486,15 +1487,26 @@ func validatePublicAddress(addr string) error { if host == "localhost" { return errors.New("localhost is not a valid address") } + ip := net.ParseIP(host) - if ip == nil { - return errors.New("not a valid IP address") - } - if ip.IsLoopback() { - return errors.New("loopback address is not a valid address") + if ip != nil { + if ip.IsLoopback() { + return errors.New("loopback address is not a valid address") + } + if ip.IsPrivate() { + return errors.New("private address is not a valid address") + } + return nil } - if ip.IsPrivate() { - return errors.New("private address is not a valid address") + + idnaValidator := idna.New( + idna.ValidateLabels(true), + idna.VerifyDNSLength(true), + idna.StrictDomainName(true), + idna.CheckHyphens(true), + ) + if _, err := idnaValidator.ToASCII(host); err != nil { + return fmt.Errorf("invalid hostname: %w", err) } return nil diff --git a/pkg/node/node_test.go b/pkg/node/node_test.go index e051c2230b1..5b5ca7ad967 100644 --- a/pkg/node/node_test.go +++ b/pkg/node/node_test.go @@ -69,9 +69,14 @@ func TestValidatePublicAddress(t *testing.T) { expErr: false, }, { - name: "invalid IP", - addr: "not-an-ip:8080", - expErr: true, + name: "valid hostname", + addr: "example.com:8080", + expErr: false, + }, + { + name: "valid hostname with hyphen", + addr: "test-example.com:8080", + expErr: false, }, { name: "private IP", @@ -79,8 +84,13 @@ func TestValidatePublicAddress(t *testing.T) { expErr: true, }, { - name: "hostname", - addr: "example.com:8080", + name: "invalid hostname format", + addr: "invalid..hostname:8080", + expErr: true, + }, + { + name: "hostname starts with hyphen", + addr: "-test.com:8080", expErr: true, }, } From 37ab9c742ceae4889b2f27d04fa381a927d60e6a Mon Sep 17 00:00:00 2001 From: Akrem Chabchoub <121046693+akrem-chabchoub@users.noreply.github.com> Date: Thu, 8 Jan 2026 15:06:14 +0100 Subject: [PATCH 66/96] fix: add validation for redundancy level in bytes and bzz handlers (#5311) --- openapi/Swarm.yaml | 37 ++------- pkg/api/api.go | 14 +++- pkg/api/bytes.go | 2 +- pkg/api/bytes_test.go | 67 +++++++++++++++++ pkg/api/bzz.go | 6 +- pkg/api/bzz_test.go | 141 +++++++++++++++++++++++++++++++++++ pkg/api/validation.go | 34 +++++++++ pkg/file/redundancy/level.go | 5 ++ 8 files changed, 268 insertions(+), 38 deletions(-) create mode 100644 pkg/api/validation.go diff --git a/openapi/Swarm.yaml b/openapi/Swarm.yaml index ce68c014a85..ab8c7a47c24 100644 --- a/openapi/Swarm.yaml +++ b/openapi/Swarm.yaml @@ -163,37 +163,12 @@ paths: tags: - Bytes parameters: - - in: header - schema: - $ref: "SwarmCommon.yaml#/components/parameters/SwarmPostageBatchId" - name: swarm-postage-batch-id - required: true - - in: header - schema: - $ref: "SwarmCommon.yaml#/components/parameters/SwarmTagParameter" - name: swarm-tag - required: false - - in: header - schema: - $ref: "SwarmCommon.yaml#/components/parameters/SwarmPinParameter" - name: swarm-pin - required: false - - in: header - schema: - $ref: "SwarmCommon.yaml#/components/parameters/SwarmDeferredUpload" - name: swarm-deferred-upload - required: false - - in: header - schema: - $ref: "SwarmCommon.yaml#/components/parameters/SwarmEncryptParameter" - name: swarm-encrypt - required: false - - in: header - schema: - $ref: "SwarmCommon.yaml#/components/parameters/SwarmRedundancyLevelParameter" - name: swarm-redundancy-level - required: false - + - $ref: "SwarmCommon.yaml#/components/parameters/SwarmPostageBatchId" + - $ref: "SwarmCommon.yaml#/components/parameters/SwarmTagParameter" + - $ref: "SwarmCommon.yaml#/components/parameters/SwarmPinParameter" + - $ref: "SwarmCommon.yaml#/components/parameters/SwarmDeferredUpload" + - $ref: "SwarmCommon.yaml#/components/parameters/SwarmEncryptParameter" + - $ref: "SwarmCommon.yaml#/components/parameters/SwarmRedundancyLevelParameter" requestBody: content: application/octet-stream: diff --git a/pkg/api/api.go b/pkg/api/api.go index 3ab9a0eabb4..2b713bc76d4 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -215,8 +215,9 @@ type Service struct { whitelistedWithdrawalAddress []common.Address - preMapHooks map[string]func(v string) (string, error) - validate *validator.Validate + preMapHooks map[string]func(v string) (string, error) + customValidationMessages map[string]func(err validator.FieldError) error + validate *validator.Validate redistributionAgent *storageincentives.Agent @@ -322,6 +323,7 @@ func New( } return name }) + s.setupValidation() s.stamperStore = stamperStore for _, v := range whitelistedWithdrawalAddress { @@ -710,11 +712,17 @@ func (s *Service) mapStructure(input, output any) func(string, log.Logger, http. case []byte: val = string(v) } + var cause error + if msgFn, ok := s.customValidationMessages[err.Tag()]; ok { + cause = msgFn(err) + } else { + cause = fmt.Errorf("want %s:%s", err.Tag(), err.Param()) + } vErrs = multierror.Append(vErrs, &validationError{ Entry: strings.ToLower(err.Field()), Value: val, - Cause: fmt.Errorf("want %s:%s", err.Tag(), err.Param()), + Cause: cause, }) } return response(vErrs.ErrorOrNil()) diff --git a/pkg/api/bytes.go b/pkg/api/bytes.go index 4c1cd891df0..945e9c3beae 100644 --- a/pkg/api/bytes.go +++ b/pkg/api/bytes.go @@ -40,7 +40,7 @@ func (s *Service) bytesUploadHandler(w http.ResponseWriter, r *http.Request) { Pin bool `map:"Swarm-Pin"` Deferred *bool `map:"Swarm-Deferred-Upload"` Encrypt bool `map:"Swarm-Encrypt"` - RLevel redundancy.Level `map:"Swarm-Redundancy-Level"` + RLevel redundancy.Level `map:"Swarm-Redundancy-Level" validate:"rLevel"` Act bool `map:"Swarm-Act"` HistoryAddress swarm.Address `map:"Swarm-Act-History-Address"` }{} diff --git a/pkg/api/bytes_test.go b/pkg/api/bytes_test.go index 40280e44992..56ec6574565 100644 --- a/pkg/api/bytes_test.go +++ b/pkg/api/bytes_test.go @@ -8,11 +8,13 @@ import ( "bytes" "context" "errors" + "fmt" "net/http" "strconv" "testing" "github.com/ethersphere/bee/v2/pkg/api" + "github.com/ethersphere/bee/v2/pkg/file/redundancy" "github.com/ethersphere/bee/v2/pkg/jsonhttp" "github.com/ethersphere/bee/v2/pkg/jsonhttp/jsonhttptest" "github.com/ethersphere/bee/v2/pkg/log" @@ -408,3 +410,68 @@ func TestBytesDirectUpload(t *testing.T) { }), ) } + +func TestBytesRedundancyLevel(t *testing.T) { + t.Parallel() + + client, _, _, _ := newTestServer(t, testServerOptions{ + Storer: mockstorer.New(), + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + const maxValidLevel = redundancy.PARANOID + + tests := []struct { + name string + level int + want *jsonhttp.StatusResponse + }{ + {"minimum level (NONE) is valid", int(redundancy.NONE), nil}, + {"maximum valid level (PARANOID) is valid", int(maxValidLevel), nil}, + { + "level below minimum is invalid", int(-1), + &jsonhttp.StatusResponse{ + Code: http.StatusBadRequest, + Message: "invalid header params", + Reasons: []jsonhttp.Reason{ + { + Field: "Swarm-Redundancy-Level", + Error: "invalid syntax", + }, + }, + }, + }, + { + "level above maximum is invalid", int(maxValidLevel + 1), + &jsonhttp.StatusResponse{ + Code: http.StatusBadRequest, + Message: "invalid header params", + Reasons: []jsonhttp.Reason{ + { + Field: "swarm-redundancy-level", + Error: fmt.Sprintf("want redundancy level to be between %d and %d", int(redundancy.NONE), int(redundancy.PARANOID)), + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + opts := []jsonhttptest.Option{ + jsonhttptest.WithRequestHeader(api.SwarmDeferredUploadHeader, "true"), + jsonhttptest.WithRequestHeader(api.SwarmPostageBatchIdHeader, batchOkStr), + jsonhttptest.WithRequestHeader(api.SwarmRedundancyLevelHeader, strconv.Itoa(tt.level)), + jsonhttptest.WithRequestBody(bytes.NewReader([]byte("test"))), + } + var statusCode int + if tt.want == nil { + statusCode = http.StatusCreated + } else { + statusCode = tt.want.Code + opts = append(opts, jsonhttptest.WithExpectedJSONResponse(*tt.want)) + } + jsonhttptest.Request(t, client, http.MethodPost, "/bytes", statusCode, opts...) + }) + } +} diff --git a/pkg/api/bzz.go b/pkg/api/bzz.go index e547f6f951d..d99594bc682 100644 --- a/pkg/api/bzz.go +++ b/pkg/api/bzz.go @@ -72,7 +72,7 @@ func (s *Service) bzzUploadHandler(w http.ResponseWriter, r *http.Request) { Deferred *bool `map:"Swarm-Deferred-Upload"` Encrypt bool `map:"Swarm-Encrypt"` IsDir bool `map:"Swarm-Collection"` - RLevel redundancy.Level `map:"Swarm-Redundancy-Level"` + RLevel redundancy.Level `map:"Swarm-Redundancy-Level" validate:"rLevel"` Act bool `map:"Swarm-Act"` HistoryAddress swarm.Address `map:"Swarm-Act-History-Address"` }{} @@ -480,7 +480,7 @@ func (s *Service) serveReference(logger log.Logger, address swarm.Address, pathV Cache *bool `map:"Swarm-Cache"` Strategy *getter.Strategy `map:"Swarm-Redundancy-Strategy"` FallbackMode *bool `map:"Swarm-Redundancy-Fallback-Mode"` - RLevel *redundancy.Level `map:"Swarm-Redundancy-Level"` + RLevel *redundancy.Level `map:"Swarm-Redundancy-Level" validate:"omitempty,rLevel"` ChunkRetrievalTimeout *string `map:"Swarm-Chunk-Retrieval-Timeout"` }{} @@ -693,7 +693,7 @@ func (s *Service) serveManifestEntry( func (s *Service) downloadHandler(logger log.Logger, w http.ResponseWriter, r *http.Request, reference swarm.Address, additionalHeaders http.Header, etag, headersOnly bool, rootCh swarm.Chunk) { headers := struct { Strategy *getter.Strategy `map:"Swarm-Redundancy-Strategy"` - RLevel *redundancy.Level `map:"Swarm-Redundancy-Level"` + RLevel *redundancy.Level `map:"Swarm-Redundancy-Level" validate:"omitempty,rLevel"` FallbackMode *bool `map:"Swarm-Redundancy-Fallback-Mode"` ChunkRetrievalTimeout *string `map:"Swarm-Chunk-Retrieval-Timeout"` LookaheadBufferSize *int `map:"Swarm-Lookahead-Buffer-Size"` diff --git a/pkg/api/bzz_test.go b/pkg/api/bzz_test.go index c12c108d418..3f444e6cb07 100644 --- a/pkg/api/bzz_test.go +++ b/pkg/api/bzz_test.go @@ -1178,3 +1178,144 @@ func TestBzzDownloadHeaders(t *testing.T) { jsonhttptest.WithExpectedResponseHeader(api.ContentTypeHeader, "text/html; charset=utf-8"), ) } + +func TestBzzUploadRedundancyLevel(t *testing.T) { + t.Parallel() + + client, _, _, _ := newTestServer(t, testServerOptions{ + Storer: mockstorer.New(), + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + const maxValidLevel = redundancy.PARANOID + + tests := []struct { + name string + level int + want *jsonhttp.StatusResponse + }{ + {"minimum level (NONE) is valid", int(redundancy.NONE), nil}, + {"maximum valid level (PARANOID) is valid", int(maxValidLevel), nil}, + { + "level below minimum is invalid", int(-1), + &jsonhttp.StatusResponse{ + Code: http.StatusBadRequest, + Message: "invalid header params", + Reasons: []jsonhttp.Reason{ + { + Field: "Swarm-Redundancy-Level", + Error: "invalid syntax", + }, + }, + }, + }, + { + "level above maximum is invalid", int(maxValidLevel + 1), + &jsonhttp.StatusResponse{ + Code: http.StatusBadRequest, + Message: "invalid header params", + Reasons: []jsonhttp.Reason{ + { + Field: "swarm-redundancy-level", + Error: fmt.Sprintf("want redundancy level to be between %d and %d", int(redundancy.NONE), int(redundancy.PARANOID)), + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + opts := []jsonhttptest.Option{ + jsonhttptest.WithRequestHeader(api.ContentTypeHeader, "text/plain"), + jsonhttptest.WithRequestHeader(api.SwarmDeferredUploadHeader, "true"), + jsonhttptest.WithRequestHeader(api.SwarmPostageBatchIdHeader, batchOkStr), + jsonhttptest.WithRequestHeader(api.SwarmRedundancyLevelHeader, strconv.Itoa(tt.level)), + jsonhttptest.WithRequestBody(bytes.NewReader([]byte("test"))), + } + var statusCode int + if tt.want == nil { + statusCode = http.StatusCreated + } else { + statusCode = tt.want.Code + opts = append(opts, jsonhttptest.WithExpectedJSONResponse(*tt.want)) + } + jsonhttptest.Request(t, client, http.MethodPost, "/bzz", statusCode, opts...) + }) + } +} + +func TestBzzDownloadRedundancyLevel(t *testing.T) { + t.Parallel() + + client, _, _, _ := newTestServer(t, testServerOptions{ + Storer: mockstorer.New(), + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + testData := []byte("test download redundancy level") + var resp api.BzzUploadResponse + jsonhttptest.Request(t, client, http.MethodPost, "/bzz", http.StatusCreated, + jsonhttptest.WithRequestHeader(api.ContentTypeHeader, "text/plain"), + jsonhttptest.WithRequestHeader(api.SwarmDeferredUploadHeader, "true"), + jsonhttptest.WithRequestHeader(api.SwarmPostageBatchIdHeader, batchOkStr), + jsonhttptest.WithRequestBody(bytes.NewReader(testData)), + jsonhttptest.WithUnmarshalJSONResponse(&resp), + ) + + const maxValidLevel = redundancy.PARANOID + + tests := []struct { + name string + level int + want *jsonhttp.StatusResponse + }{ + {"minimum level (NONE) is valid", int(redundancy.NONE), nil}, + {"maximum valid level (PARANOID) is valid", int(maxValidLevel), nil}, + { + "level below minimum is invalid", int(-1), + &jsonhttp.StatusResponse{ + Code: http.StatusBadRequest, + Message: "invalid header params", + Reasons: []jsonhttp.Reason{ + { + Field: "Swarm-Redundancy-Level", + Error: "invalid syntax", + }, + }, + }, + }, + { + "level above maximum is invalid", int(maxValidLevel + 1), + &jsonhttp.StatusResponse{ + Code: http.StatusBadRequest, + Message: "invalid header params", + Reasons: []jsonhttp.Reason{ + { + Field: "swarm-redundancy-level", + Error: fmt.Sprintf("want redundancy level to be between %d and %d", int(redundancy.NONE), int(redundancy.PARANOID)), + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + opts := []jsonhttptest.Option{ + jsonhttptest.WithRequestHeader(api.SwarmRedundancyLevelHeader, strconv.Itoa(tt.level)), + } + var statusCode int + if tt.want == nil { + statusCode = http.StatusOK + opts = append(opts, + jsonhttptest.WithExpectedResponse(testData), + ) + } else { + statusCode = tt.want.Code + opts = append(opts, jsonhttptest.WithExpectedJSONResponse(*tt.want)) + } + jsonhttptest.Request(t, client, http.MethodGet, "/bzz/"+resp.Reference.String(), statusCode, opts...) + }) + } +} diff --git a/pkg/api/validation.go b/pkg/api/validation.go new file mode 100644 index 00000000000..2e1422ff414 --- /dev/null +++ b/pkg/api/validation.go @@ -0,0 +1,34 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package api + +import ( + "fmt" + + "github.com/ethersphere/bee/v2/pkg/file/redundancy" + "github.com/go-playground/validator/v10" +) + +const ( + RedundancyLevelTag = "rLevel" +) + +// setupValidation configures custom validation rules and their custom error messages. +func (s *Service) setupValidation() { + err := s.validate.RegisterValidation(RedundancyLevelTag, func(fl validator.FieldLevel) bool { + level := redundancy.Level(fl.Field().Uint()) + return level.Validate() + }) + if err != nil { + s.logger.Error(err, "failed to register validation") + panic(err) + } + + s.customValidationMessages = map[string]func(err validator.FieldError) error{ + RedundancyLevelTag: func(err validator.FieldError) error { + return fmt.Errorf("want redundancy level to be between %d and %d", int(redundancy.NONE), int(redundancy.PARANOID)) + }, + } +} diff --git a/pkg/file/redundancy/level.go b/pkg/file/redundancy/level.go index 411da15ec98..c4eeacddc9f 100644 --- a/pkg/file/redundancy/level.go +++ b/pkg/file/redundancy/level.go @@ -29,6 +29,11 @@ const ( PARANOID ) +// Validate validates the redundancy level +func (l Level) Validate() bool { + return l >= NONE && l <= PARANOID +} + // GetParities returns number of parities based on appendix F table 5 func (l Level) GetParities(shards int) int { et, err := l.getErasureTable() From 66519d743fd768d066bb19858c2f7fe5245b7b38 Mon Sep 17 00:00:00 2001 From: Akrem Chabchoub <121046693+akrem-chabchoub@users.noreply.github.com> Date: Thu, 8 Jan 2026 15:08:04 +0100 Subject: [PATCH 67/96] test(libp2p): update reacher tests to use synctest (#5267) --- .../libp2p/internal/reacher/reacher_test.go | 91 ++++++++++--------- 1 file changed, 47 insertions(+), 44 deletions(-) diff --git a/pkg/p2p/libp2p/internal/reacher/reacher_test.go b/pkg/p2p/libp2p/internal/reacher/reacher_test.go index a0e55f2fb35..515e05eab88 100644 --- a/pkg/p2p/libp2p/internal/reacher/reacher_test.go +++ b/pkg/p2p/libp2p/internal/reacher/reacher_test.go @@ -10,6 +10,8 @@ import ( "testing" "time" + "testing/synctest" + "github.com/ethersphere/bee/v2/pkg/log" "github.com/ethersphere/bee/v2/pkg/p2p" "github.com/ethersphere/bee/v2/pkg/p2p/libp2p/internal/reacher" @@ -63,65 +65,66 @@ func TestPingSuccess(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - t.Parallel() - - done := make(chan struct{}) - mock := newMock(tc.pingFunc, tc.reachableFunc(done)) + synctest.Test(t, func(t *testing.T) { + done := make(chan struct{}) + mock := newMock(tc.pingFunc, tc.reachableFunc(done)) - r := reacher.New(mock, mock, &defaultOptions, log.Noop) - testutil.CleanupCloser(t, r) + r := reacher.New(mock, mock, &defaultOptions, log.Noop) + testutil.CleanupCloser(t, r) - overlay := swarm.RandAddress(t) - addr, _ := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/7071/p2p/16Uiu2HAmTBuJT9LvNmBiQiNoTsxE5mtNy6YG3paw79m94CRa9sRb") + overlay := swarm.RandAddress(t) + addr, _ := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/7071/p2p/16Uiu2HAmTBuJT9LvNmBiQiNoTsxE5mtNy6YG3paw79m94CRa9sRb") - r.Connected(overlay, addr) + r.Connected(overlay, addr) - select { - case <-time.After(time.Second * 5): - t.Fatalf("test timed out") - case <-done: - } + select { + case <-time.After(time.Second * 5): + t.Fatalf("test timed out") + case <-done: + } + }) }) } } func TestDisconnected(t *testing.T) { - t.Parallel() - - var ( - disconnectedOverlay = swarm.RandAddress(t) - disconnectedMa, _ = ma.NewMultiaddr("/ip4/127.0.0.1/tcp/7071/p2p/16Uiu2HAmTBuJT9LvNmBiQiNoTsxE5mtNy6YG3paw79m94CRa9sRb") - ) - - /* - Because the Disconnected is called after Connected, it may be that one of the workers - have picked up the peer already. So to test that the Disconnected really works, - if the ping function pings the peer we are trying to disconnect, we return an error - which triggers another attempt in the future, which by the, the peer should already be removed. - */ - var errs atomic.Int64 - pingFunc := func(_ context.Context, a ma.Multiaddr) (time.Duration, error) { - if a != nil && a.Equal(disconnectedMa) { - errs.Inc() - if errs.Load() > 1 { - t.Fatalf("overlay should be disconnected already") + synctest.Test(t, func(t *testing.T) { + var ( + disconnectedOverlay = swarm.RandAddress(t) + disconnectedMa, _ = ma.NewMultiaddr("/ip4/127.0.0.1/tcp/7071/p2p/16Uiu2HAmTBuJT9LvNmBiQiNoTsxE5mtNy6YG3paw79m94CRa9sRb") + ) + + /* + Because the Disconnected is called after Connected, it may be that one of the workers + have picked up the peer already. So to test that the Disconnected really works, + if the ping function pings the peer we are trying to disconnect, we return an error + which triggers another attempt in the future, which by the, the peer should already be removed. + */ + + var errs atomic.Int64 + pingFunc := func(_ context.Context, a ma.Multiaddr) (time.Duration, error) { + if a != nil && a.Equal(disconnectedMa) { + errs.Inc() + if errs.Load() > 1 { + t.Fatalf("overlay should be disconnected already") + } + return 0, errors.New("test error") } - return 0, errors.New("test error") + return 0, nil } - return 0, nil - } - reachableFunc := func(addr swarm.Address, b p2p.ReachabilityStatus) {} + reachableFunc := func(addr swarm.Address, b p2p.ReachabilityStatus) {} - mock := newMock(pingFunc, reachableFunc) + mock := newMock(pingFunc, reachableFunc) - r := reacher.New(mock, mock, &defaultOptions, log.Noop) - testutil.CleanupCloser(t, r) + r := reacher.New(mock, mock, &defaultOptions, log.Noop) + testutil.CleanupCloser(t, r) - addr, _ := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/7072/p2p/16Uiu2HAmTBuJT9LvNmBiQiNoTsxE5mtNy6YG3paw79m94CRa9sRb") - r.Connected(swarm.RandAddress(t), addr) - r.Connected(disconnectedOverlay, disconnectedMa) - r.Disconnected(disconnectedOverlay) + addr, _ := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/7072/p2p/16Uiu2HAmTBuJT9LvNmBiQiNoTsxE5mtNy6YG3paw79m94CRa9sRb") + r.Connected(swarm.RandAddress(t), addr) + r.Connected(disconnectedOverlay, disconnectedMa) + r.Disconnected(disconnectedOverlay) + }) } type mock struct { From c92d8c58721fb32e1221aefeeba970942b738501 Mon Sep 17 00:00:00 2001 From: Calin M Date: Mon, 12 Jan 2026 18:02:06 +0200 Subject: [PATCH 68/96] feat(p2p): add AutoTLS support for secure WebSocket connections (#5187) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Janoš Guljaš Co-authored-by: Ljubisa Gacevic Co-authored-by: Ljubiša Gačević <35105035+gacevicljubisa@users.noreply.github.com> --- .github/workflows/go.yml | 9 + .gitignore | 1 + cmd/bee/cmd/cmd.go | 13 + cmd/bee/cmd/start.go | 7 + go.mod | 148 +++-- go.sum | 383 +++++------ packaging/bee.yaml | 12 + packaging/homebrew-amd64/bee.yaml | 12 + packaging/homebrew-arm64/bee.yaml | 12 + packaging/scoop/bee.yaml | 12 + pkg/api/peer_test.go | 3 - pkg/api/util.go | 32 +- pkg/api/util_test.go | 14 + pkg/bzz/underlay_test.go | 29 + pkg/hive/hive.go | 122 ++-- pkg/hive/hive_test.go | 263 +++++++- pkg/node/bootstrap.go | 21 +- pkg/node/node.go | 36 +- pkg/p2p/libp2p/composite_resolver_test.go | 162 +++++ pkg/p2p/libp2p/connections_test.go | 130 +++- pkg/p2p/libp2p/export_test.go | 14 + pkg/p2p/libp2p/libp2p.go | 593 +++++++++++++----- pkg/p2p/libp2p/libp2p_test.go | 21 +- pkg/p2p/libp2p/libp2ptest/buffer.go | 28 + pkg/p2p/libp2p/libp2ptest/service.go | 69 ++ pkg/p2p/libp2p/mock/mock_certmagic.go | 158 +++++ pkg/p2p/libp2p/mock/mock_certmagic_test.go | 112 ++++ pkg/p2p/libp2p/mock/testdata/cert.pem | 19 + pkg/p2p/libp2p/mock/testdata/key.pem | 28 + pkg/p2p/libp2p/protocols_test.go | 37 +- pkg/p2p/libp2p/static_resolver.go | 60 +- pkg/p2p/libp2p/static_resolver_test.go | 38 +- pkg/p2p/libp2p/upnp.go | 68 -- pkg/puller/puller.go | 12 +- pkg/pullsync/pullsync.go | 23 +- pkg/pushsync/main_test.go | 2 +- pkg/pushsync/pushsync.go | 5 +- pkg/pushsync/pushsync_integration_test.go | 201 ++++++ pkg/retrieval/main_test.go | 1 + pkg/retrieval/retrieval.go | 5 +- pkg/retrieval/retrieval_integration_test.go | 153 +++++ .../swap/swapprotocol/swapprotocol.go | 26 +- pkg/topology/kademlia/kademlia.go | 13 +- pkg/topology/kademlia/kademlia_test.go | 69 ++ 44 files changed, 2468 insertions(+), 708 deletions(-) create mode 100644 pkg/p2p/libp2p/composite_resolver_test.go create mode 100644 pkg/p2p/libp2p/libp2ptest/buffer.go create mode 100644 pkg/p2p/libp2p/libp2ptest/service.go create mode 100644 pkg/p2p/libp2p/mock/mock_certmagic.go create mode 100644 pkg/p2p/libp2p/mock/mock_certmagic_test.go create mode 100644 pkg/p2p/libp2p/mock/testdata/cert.pem create mode 100644 pkg/p2p/libp2p/mock/testdata/key.pem delete mode 100644 pkg/p2p/libp2p/upnp.go create mode 100644 pkg/pushsync/pushsync_integration_test.go create mode 100644 pkg/retrieval/retrieval_integration_test.go diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 6e548e238e2..4bbfa5be9bd 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -27,6 +27,15 @@ jobs: with: cache: true go-version-file: go.mod + - name: Increase UDP buffer sizes (Ubuntu) + if: matrix.os == 'ubuntu-latest' + run: | + sudo sysctl -w net.core.rmem_max=7500000 + sudo sysctl -w net.core.wmem_max=7500000 + - name: Increase UDP buffer sizes (macOS) + if: matrix.os == 'macos-latest' + run: | + sudo sysctl -w kern.ipc.maxsockbuf=6291456 - name: Build run: make build - name: Test with race detector (Ubuntu and MacOS) diff --git a/.gitignore b/.gitignore index b7b0629ab25..c94f81d64ab 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ /.idea /.vscode /tmp +/vendor # Compiled Object files, Static and Dynamic libs (Shared Objects) *.o diff --git a/cmd/bee/cmd/cmd.go b/cmd/bee/cmd/cmd.go index 5b2c156c630..9324d58012f 100644 --- a/cmd/bee/cmd/cmd.go +++ b/cmd/bee/cmd/cmd.go @@ -17,6 +17,7 @@ import ( "github.com/ethersphere/bee/v2/pkg/log" "github.com/ethersphere/bee/v2/pkg/node" "github.com/ethersphere/bee/v2/pkg/swarm" + p2pforge "github.com/ipshipyard/p2p-forge/client" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -82,6 +83,12 @@ const ( optionReserveCapacityDoubling = "reserve-capacity-doubling" optionSkipPostageSnapshot = "skip-postage-snapshot" optionNameMinimumGasTipCap = "minimum-gas-tip-cap" + optionNameP2PWSSEnable = "p2p-wss-enable" + optionP2PWSSAddr = "p2p-wss-addr" + optionNATWSSAddr = "nat-wss-addr" + optionAutoTLSDomain = "autotls-domain" + optionAutoTLSRegistrationEndpoint = "autotls-registration-endpoint" + optionAutoTLSCAEndpoint = "autotls-ca-endpoint" ) // nolint:gochecknoinits @@ -292,6 +299,12 @@ func (c *command) setAllFlags(cmd *cobra.Command) { cmd.Flags().Int(optionReserveCapacityDoubling, 0, "reserve capacity doubling") cmd.Flags().Bool(optionSkipPostageSnapshot, false, "skip postage snapshot") cmd.Flags().Uint64(optionNameMinimumGasTipCap, 0, "minimum gas tip cap in wei for transactions, 0 means use suggested gas tip cap") + cmd.Flags().Bool(optionNameP2PWSSEnable, false, "Enable Secure WebSocket P2P connections") + cmd.Flags().String(optionP2PWSSAddr, ":1635", "p2p wss address") + cmd.Flags().String(optionNATWSSAddr, "", "WSS NAT exposed address") + cmd.Flags().String(optionAutoTLSDomain, p2pforge.DefaultForgeDomain, "autotls domain") + cmd.Flags().String(optionAutoTLSRegistrationEndpoint, p2pforge.DefaultForgeEndpoint, "autotls registration endpoint") + cmd.Flags().String(optionAutoTLSCAEndpoint, p2pforge.DefaultCAEndpoint, "autotls certificate authority endpoint") } func newLogger(cmd *cobra.Command, verbosity string) (log.Logger, error) { diff --git a/cmd/bee/cmd/start.go b/cmd/bee/cmd/start.go index c6662bcba57..9633b9bd6f1 100644 --- a/cmd/bee/cmd/start.go +++ b/cmd/bee/cmd/start.go @@ -281,12 +281,16 @@ func buildBeeNode(ctx context.Context, c *command, cmd *cobra.Command, logger lo Addr: c.config.GetString(optionNameP2PAddr), AllowPrivateCIDRs: c.config.GetBool(optionNameAllowPrivateCIDRs), APIAddr: c.config.GetString(optionNameAPIAddr), + EnableWSS: c.config.GetBool(optionNameP2PWSSEnable), + WSSAddr: c.config.GetString(optionP2PWSSAddr), + AutoTLSStorageDir: filepath.Join(c.config.GetString(optionNameDataDir), "autotls"), BlockchainRpcEndpoint: c.config.GetString(optionNameBlockchainRpcEndpoint), BlockProfile: c.config.GetBool(optionNamePProfBlock), BlockTime: networkConfig.blockTime, BootnodeMode: bootNode, Bootnodes: networkConfig.bootNodes, CacheCapacity: c.config.GetUint64(optionNameCacheCapacity), + AutoTLSCAEndpoint: c.config.GetString(optionAutoTLSCAEndpoint), ChainID: networkConfig.chainID, ChequebookEnable: c.config.GetBool(optionNameChequebookEnable), CORSAllowedOrigins: c.config.GetStringSlice(optionCORSAllowedOrigins), @@ -297,12 +301,15 @@ func buildBeeNode(ctx context.Context, c *command, cmd *cobra.Command, logger lo DBWriteBufferSize: c.config.GetUint64(optionNameDBWriteBufferSize), EnableStorageIncentives: c.config.GetBool(optionNameStorageIncentivesEnable), EnableWS: c.config.GetBool(optionNameP2PWSEnable), + AutoTLSDomain: c.config.GetString(optionAutoTLSDomain), + AutoTLSRegistrationEndpoint: c.config.GetString(optionAutoTLSRegistrationEndpoint), FullNodeMode: fullNode, Logger: logger, MinimumGasTipCap: c.config.GetUint64(optionNameMinimumGasTipCap), MinimumStorageRadius: c.config.GetUint(optionMinimumStorageRadius), MutexProfile: c.config.GetBool(optionNamePProfMutex), NATAddr: c.config.GetString(optionNameNATAddr), + NATWSSAddr: c.config.GetString(optionNATWSSAddr), NeighborhoodSuggester: neighborhoodSuggester, PaymentEarly: c.config.GetInt64(optionNamePaymentEarly), PaymentThreshold: c.config.GetString(optionNamePaymentThreshold), diff --git a/go.mod b/go.mod index a55a0923d50..1cee528864c 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( contrib.go.opencensus.io/exporter/prometheus v0.4.2 github.com/armon/go-radix v1.0.0 github.com/btcsuite/btcd/btcec/v2 v2.3.2 + github.com/caddyserver/certmagic v0.21.6 github.com/coreos/go-semver v0.3.0 github.com/ethereum/go-ethereum v1.15.11 github.com/ethersphere/batch-archive v0.0.4 @@ -15,33 +16,33 @@ require ( github.com/ethersphere/go-storage-incentives-abi v0.9.4 github.com/ethersphere/go-sw3-abi v0.6.9 github.com/ethersphere/langos v1.0.0 - github.com/go-playground/validator/v10 v10.11.1 + github.com/go-playground/validator/v10 v10.19.0 github.com/gogo/protobuf v1.3.2 - github.com/google/go-cmp v0.6.0 - github.com/google/uuid v1.4.0 + github.com/google/go-cmp v0.7.0 + github.com/google/uuid v1.6.0 github.com/gorilla/handlers v1.4.2 - github.com/gorilla/mux v1.8.0 - github.com/gorilla/websocket v1.5.1 + github.com/gorilla/mux v1.8.1 + github.com/gorilla/websocket v1.5.3 github.com/hashicorp/go-multierror v1.1.1 - github.com/hashicorp/golang-lru/v2 v2.0.5 - github.com/ipfs/go-cid v0.4.1 + github.com/hashicorp/golang-lru/v2 v2.0.7 + github.com/ipfs/go-cid v0.5.0 + github.com/ipshipyard/p2p-forge v0.7.0 github.com/kardianos/service v1.2.2 github.com/klauspost/reedsolomon v1.11.8 - github.com/libp2p/go-libp2p v0.33.2 - github.com/multiformats/go-multiaddr v0.12.3 - github.com/multiformats/go-multiaddr-dns v0.3.1 + github.com/libp2p/go-libp2p v0.46.0 + github.com/multiformats/go-multiaddr v0.16.1 + github.com/multiformats/go-multiaddr-dns v0.4.1 github.com/multiformats/go-multihash v0.2.3 - github.com/multiformats/go-multistream v0.5.0 - github.com/multiformats/go-varint v0.0.7 + github.com/multiformats/go-multistream v0.6.1 github.com/opentracing/opentracing-go v1.2.0 - github.com/prometheus/client_golang v1.21.1 + github.com/prometheus/client_golang v1.22.0 github.com/spf13/afero v1.6.0 github.com/spf13/cobra v1.8.1 github.com/spf13/viper v1.7.0 - github.com/stretchr/testify v1.10.0 + github.com/stretchr/testify v1.11.1 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 github.com/uber/jaeger-client-go v2.24.0+incompatible - github.com/vmihailenco/msgpack/v5 v5.3.4 + github.com/vmihailenco/msgpack/v5 v5.4.1 github.com/wealdtech/go-ens/v3 v3.5.1 gitlab.com/nolash/go-mockbytes v0.0.7 go.uber.org/atomic v1.11.0 @@ -51,7 +52,7 @@ require ( golang.org/x/sync v0.18.0 golang.org/x/sys v0.38.0 golang.org/x/term v0.37.0 - golang.org/x/time v0.9.0 + golang.org/x/time v0.12.0 gopkg.in/yaml.v2 v2.4.0 resenje.org/feed v0.1.2 resenje.org/multex v0.1.0 @@ -59,71 +60,62 @@ require ( resenje.org/web v0.4.3 ) -require golang.org/x/telemetry v0.0.0-20251008203120-078029d740a8 // indirect - require ( - github.com/BurntSushi/toml v1.1.0 // indirect + github.com/BurntSushi/toml v1.3.2 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/StackExchange/wmi v1.2.1 // indirect github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.20.0 // indirect github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 // indirect + github.com/caddyserver/zerossl v0.1.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/codahale/hdrhistogram v0.0.0-00010101000000-000000000000 // indirect github.com/consensys/gnark-crypto v0.18.1 // indirect - github.com/containerd/cgroups v1.1.0 // indirect - github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/crate-crypto/go-eth-kzg v1.3.0 // indirect github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect github.com/deckarep/golang-set/v2 v2.6.0 // indirect - github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect - github.com/docker/go-units v0.5.0 // indirect - github.com/elastic/gosigar v0.14.2 // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect github.com/ethereum/c-kzg-4844/v2 v2.1.0 // indirect github.com/ethereum/go-verkle v0.2.2 // indirect github.com/felixge/fgprof v0.9.5 github.com/flynn/noise v1.1.0 // indirect - github.com/francoispqt/gojay v1.2.13 // indirect - github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/go-kit/log v0.2.1 // indirect - github.com/go-logfmt/logfmt v0.5.1 // indirect + github.com/go-logfmt/logfmt v0.6.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect - github.com/go-playground/locales v0.14.0 // indirect - github.com/go-playground/universal-translator v0.18.0 // indirect - github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect - github.com/godbus/dbus/v5 v5.1.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect - github.com/google/gopacket v1.1.19 // indirect - github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 // indirect - github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect - github.com/hashicorp/errwrap v1.0.0 // indirect + github.com/google/pprof v0.0.0-20250607225305-033d6d78b36a // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/holiman/uint256 v1.3.2 // indirect github.com/huin/goupnp v1.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/ipfs/go-log/v2 v2.5.1 // indirect + github.com/ipfs/go-log/v2 v2.6.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect - github.com/klauspost/compress v1.17.11 // indirect - github.com/klauspost/cpuid/v2 v2.2.7 // indirect - github.com/koron/go-ssdp v0.0.4 // indirect - github.com/leodido/go-urn v1.2.1 // indirect + github.com/klauspost/cpuid/v2 v2.2.10 // indirect + github.com/koron/go-ssdp v0.0.6 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + github.com/libdns/libdns v0.2.2 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect - github.com/libp2p/go-flow-metrics v0.1.0 // indirect + github.com/libp2p/go-flow-metrics v0.2.0 // indirect github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect github.com/libp2p/go-msgio v0.3.0 // indirect - github.com/libp2p/go-nat v0.2.0 // indirect - github.com/libp2p/go-netroute v0.2.1 // indirect + github.com/libp2p/go-netroute v0.3.0 // indirect github.com/libp2p/go-reuseport v0.4.0 // indirect - github.com/libp2p/go-yamux/v4 v4.0.1 // indirect + github.com/libp2p/go-yamux/v5 v5.0.1 // indirect github.com/magiconair/properties v1.8.1 // indirect github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/miekg/dns v1.1.58 // indirect + github.com/mholt/acmez/v3 v3.0.0 // indirect + github.com/miekg/dns v1.1.66 // indirect github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect github.com/minio/sha256-simd v1.0.1 // indirect @@ -133,24 +125,42 @@ require ( github.com/multiformats/go-base36 v0.2.0 // indirect github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect github.com/multiformats/go-multibase v0.2.0 // indirect - github.com/multiformats/go-multicodec v0.9.0 // indirect + github.com/multiformats/go-multicodec v0.9.1 // indirect + github.com/multiformats/go-varint v0.0.7 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/onsi/ginkgo/v2 v2.15.0 // indirect - github.com/opencontainers/runtime-spec v1.2.0 // indirect + github.com/onsi/gomega v1.36.3 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/pelletier/go-toml v1.8.0 // indirect + github.com/pion/datachannel v1.5.10 // indirect + github.com/pion/dtls/v2 v2.2.12 // indirect + github.com/pion/dtls/v3 v3.0.6 // indirect + github.com/pion/ice/v4 v4.0.10 // indirect + github.com/pion/interceptor v0.1.40 // indirect + github.com/pion/logging v0.2.3 // indirect + github.com/pion/mdns/v2 v2.0.7 // indirect + github.com/pion/randutil v0.1.0 // indirect + github.com/pion/rtcp v1.2.15 // indirect + github.com/pion/rtp v1.8.19 // indirect + github.com/pion/sctp v1.8.39 // indirect + github.com/pion/sdp/v3 v3.0.13 // indirect + github.com/pion/srtp/v3 v3.0.6 // indirect + github.com/pion/stun v0.6.1 // indirect + github.com/pion/stun/v3 v3.0.0 // indirect + github.com/pion/transport/v2 v2.2.10 // indirect + github.com/pion/transport/v3 v3.0.7 // indirect + github.com/pion/turn/v4 v4.0.2 // indirect + github.com/pion/webrtc/v4 v4.1.2 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.62.0 - github.com/prometheus/procfs v0.15.1 // indirect - github.com/prometheus/statsd_exporter v0.22.7 // indirect - github.com/quic-go/qpack v0.4.0 // indirect - github.com/quic-go/quic-go v0.42.0 // indirect - github.com/quic-go/webtransport-go v0.6.0 // indirect - github.com/raulk/go-watchdog v1.3.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.64.0 + github.com/prometheus/procfs v0.16.1 // indirect + github.com/prometheus/statsd_exporter v0.26.1 // indirect + github.com/quic-go/qpack v0.6.0 // indirect + github.com/quic-go/quic-go v0.57.1 // indirect + github.com/quic-go/webtransport-go v0.9.0 // indirect + github.com/rogpeppe/go-internal v1.13.1 // indirect github.com/shirou/gopsutil v3.21.5+incompatible // indirect - github.com/smartystreets/assertions v1.1.1 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/cast v1.3.0 // indirect github.com/spf13/jwalterweatherman v1.0.0 // indirect @@ -162,20 +172,24 @@ require ( github.com/uber/jaeger-lib v2.2.0+incompatible // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/wealdtech/go-multicodec v1.4.0 // indirect + github.com/wlynxg/anet v0.0.5 // indirect + github.com/zeebo/blake3 v0.2.4 // indirect go.opencensus.io v0.24.0 // indirect - go.uber.org/dig v1.17.1 // indirect - go.uber.org/fx v1.20.1 // indirect - go.uber.org/mock v0.4.0 // indirect + go.uber.org/dig v1.19.0 // indirect + go.uber.org/fx v1.24.0 // indirect + go.uber.org/mock v0.5.2 // indirect go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.27.0 // indirect - golang.org/x/exp v0.0.0-20240213143201-ec583247a57a // indirect + go.uber.org/zap v1.27.0 + go.uber.org/zap/exp v0.3.0 // indirect + golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 // indirect golang.org/x/mod v0.29.0 // indirect + golang.org/x/telemetry v0.0.0-20251008203120-078029d740a8 // indirect golang.org/x/text v0.31.0 // indirect golang.org/x/tools v0.38.0 // indirect - google.golang.org/protobuf v1.36.1 // indirect - gopkg.in/ini.v1 v1.57.0 // indirect + google.golang.org/protobuf v1.36.6 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - lukechampine.com/blake3 v1.2.1 // indirect + lukechampine.com/blake3 v1.4.1 // indirect ) replace github.com/codahale/hdrhistogram => github.com/HdrHistogram/hdrhistogram-go v0.0.0-20200919145931-8dac23c8dac1 diff --git a/go.sum b/go.sum index 3541cb2be1c..7b0f87f302a 100644 --- a/go.sum +++ b/go.sum @@ -59,8 +59,8 @@ github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v1.1.0 h1:ksErzDEI1khOiGPgpwuI7x2ebx/uXQNw7xJpn9Eq1+I= -github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= +github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= @@ -105,8 +105,6 @@ github.com/aws/aws-sdk-go-v2/service/route53 v1.1.1/go.mod h1:rLiOUrPLW/Er5kRcQ7 github.com/aws/aws-sdk-go-v2/service/sso v1.1.1/go.mod h1:SuZJxklHxLAXgLTc1iFXbEWkXs7QRTQpCLGaKIprQW0= github.com/aws/aws-sdk-go-v2/service/sts v1.1.1/go.mod h1:Wi0EBZwiz/K44YliU0EKxqTCJGUfYTWXrrBwkq736bM= github.com/aws/smithy-go v1.1.0/go.mod h1:EzMw8dbp/YJL4A5/sbhGddag+NPT7q084agLbB9LgIw= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -138,6 +136,10 @@ github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtE github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= +github.com/caddyserver/certmagic v0.21.6 h1:1th6GfprVfsAtFNOu4StNMF5IxK5XiaI0yZhAHlZFPE= +github.com/caddyserver/certmagic v0.21.6/go.mod h1:n1sCo7zV1Ez2j+89wrzDxo4N/T1Ws/Vx8u5NvuBFabw= +github.com/caddyserver/zerossl v0.1.3 h1:onS+pxp3M8HnHpN5MMbOMyNjmTheJyWRaZYwn+YTAyA= +github.com/caddyserver/zerossl v0.1.3/go.mod h1:CxA0acn7oEGO6//4rtrRjYgEoa4MFw/XofZnrYwGqG4= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= github.com/cespare/cp v1.1.1 h1:nCb6ZLdB7NRaqsm91JtQTAme2SKJzXVsdPIPkyJr1MU= @@ -158,7 +160,6 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8= -github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3hQ7C/YWzIGLeu5c304= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= @@ -178,21 +179,14 @@ github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/ github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1:815PAHg3wvysy0SyIqanF8gZ0Y1wjk/hrDHD/iT88+Q= github.com/consensys/gnark-crypto v0.18.1 h1:RyLV6UhPRoYYzaFnPQA4qK3DyuDgkTgskDdoGqFt3fI= github.com/consensys/gnark-crypto v0.18.1/go.mod h1:L3mXGFTe1ZN+RSJ+CLjUt9x7PNdx8ubaYfDROyp2Z8c= -github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= -github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= -github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= -github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= -github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB0alcyc= github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= @@ -206,18 +200,19 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c h1:pFUpOrbxDR6AkioZ1ySsx5yxlDQZ8stG2b88gTPxgJU= github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6UhI8N9EjYm1c2odKpFpAYeR8dsBeM7PtzQhRgxRr9U= github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= github.com/deckarep/golang-set v1.7.1/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= github.com/deckarep/golang-set/v2 v2.6.0 h1:XfcQbWM1LlMB8BsJ8N9vW5ehnnPVIw0je80NsVHagjM= github.com/deckarep/golang-set/v2 v2.6.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= -github.com/decred/dcrd/crypto/blake256 v1.0.1 h1:7PltbUIQB7u/FfZ39+DGa/ShuMyJ5ilcvdfma9wOH6Y= -github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= +github.com/decred/dcrd/crypto/blake256 v1.1.0 h1:zPMNGQCm0g4QTY27fOCorQW7EryeQ/U0x++OzVrdms8= +github.com/decred/dcrd/crypto/blake256 v1.1.0/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 h1:NMZiJj8QnKe1LgsbDayM4UoHwbvwDRwnI3hwNaAHRnc= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/deepmap/oapi-codegen v1.6.0 h1:w/d1ntwh91XI0b/8ja7+u5SvA4IFfM0UNNLmiDR1gg0= github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M= @@ -226,16 +221,10 @@ github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMa github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= -github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/elastic/gosigar v0.12.0/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= -github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/4= -github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -267,12 +256,13 @@ github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI github.com/flynn/noise v1.1.0 h1:KjPQoQCEFdZDiP03phOvGi11+SVVhBG2wOWAorLsstg= github.com/flynn/noise v1.1.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= -github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= -github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= +github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= github.com/gballet/go-libpcsclite v0.0.0-20191108122812-4678299bea08 h1:f6D9Hr8xV8uYKlyuj8XIruxlh9WjVjdh1gIicAS7ays= github.com/gballet/go-libpcsclite v0.0.0-20191108122812-4678299bea08/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= @@ -295,34 +285,27 @@ github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBj github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= -github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= -github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= +github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= -github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= -github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= -github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= -github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= -github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= -github.com/go-playground/validator/v10 v10.11.1 h1:prmOlTVv+YjZjmRmNSF3VmspqJIxJWXmqUsHwfTRRkQ= -github.com/go-playground/validator/v10 v10.11.1/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.19.0 h1:ol+5Fu+cSq9JD7SoSqe04GMI92cbn0+wvQ3bZ8b/AU4= +github.com/go-playground/validator/v10 v10.19.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY= -github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= -github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= @@ -366,8 +349,6 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -384,21 +365,18 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= -github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -408,28 +386,27 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 h1:y3N7Bm7Y9/CtpiVkw/ZWj6lSlDF3F74SfKwfTCer72Q= github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/google/pprof v0.0.0-20250607225305-033d6d78b36a h1://KbezygeMJZCSHH+HgUZiTeSoiuFspbMg1ge+eFj18= +github.com/google/pprof v0.0.0-20250607225305-033d6d78b36a/go.mod h1:5hDyRhoBCxViHszMt12TnOpEI4VVi+U8Gm9iphldiMA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.5/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= -github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 h1:l5lAOZEym3oK3SQ2HBHWsJUfbNBiTXJDeW2QDxw9AQ0= -github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/handlers v1.4.2 h1:0QniY0USkHQ1RGCLfKxeNHK9bkDHGRYGNDFBCS+YARg= github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= -github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= -github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= -github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/graph-gophers/graphql-go v1.3.0 h1:Eb9x/q6MFpCLz7jBCiP/WTxjSDrYLR1QY41SORZyNJ0= github.com/graph-gophers/graphql-go v1.3.0/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= @@ -440,8 +417,9 @@ github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpg github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE= github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= @@ -459,8 +437,8 @@ github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/golang-lru/v2 v2.0.5 h1:wW7h1TG88eUIJ2i69gaE3uNVtEPIagzhGvHgwfx2Vm4= -github.com/hashicorp/golang-lru/v2 v2.0.5/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= @@ -499,10 +477,12 @@ github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6/go.mod h1:bS github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mqiSBE6Ffsg94weZZ2c+v/ciT8QRHFOap7EKDrR0= github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1:Wbbw6tYNvwa5dlB6304Sd+82Z3f7PmVZHVKU637d4po= github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= -github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= -github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= -github.com/ipfs/go-log/v2 v2.5.1 h1:1XdUzF7048prq4aBjDQQ4SL5RxftpRGdXhNRwKSAlcY= -github.com/ipfs/go-log/v2 v2.5.1/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI= +github.com/ipfs/go-cid v0.5.0 h1:goEKKhaGm0ul11IHA7I6p1GmKz8kEYniqFopaB5Otwg= +github.com/ipfs/go-cid v0.5.0/go.mod h1:0L7vmeNXpQpUS9vt+yEARkJ8rOg43DF3iPgn4GIN0mk= +github.com/ipfs/go-log/v2 v2.6.0 h1:2Nu1KKQQ2ayonKp4MPo6pXCjqw1ULc9iohRqWV5EYqg= +github.com/ipfs/go-log/v2 v2.6.0/go.mod h1:p+Efr3qaY5YXpx9TX7MoLCSEZX5boSWj9wh86P5HJa8= +github.com/ipshipyard/p2p-forge v0.7.0 h1:PQayexxZC1FR2Vx0XOSbmZ6wDPliidS48I+xXWuF+YU= +github.com/ipshipyard/p2p-forge v0.7.0/go.mod h1:i2wg0p7WmHGyo5vYaK9COZBp8BN5Drncfu3WoQNZlQY= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= @@ -526,7 +506,6 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jsternberg/zap-logfmt v1.0.0/go.mod h1:uvPs/4X51zdkcm5jXl5SYoN+4RK21K8mysFmDaM/h+o= -github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= @@ -543,26 +522,25 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= -github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.6/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= -github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= +github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/klauspost/reedsolomon v1.11.8 h1:s8RpUW5TK4hjr+djiOpbZJB4ksx+TdYbRH7vHQpwPOY= github.com/klauspost/reedsolomon v1.11.8/go.mod h1:4bXRN+cVzMdml6ti7qLouuYi32KHJ5MGv0Qd8a47h6A= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/koron/go-ssdp v0.0.4 h1:1IDwrghSKYM7yLf7XCzbByg2sJ/JcNOZRXS2jczTwz0= -github.com/koron/go-ssdp v0.0.4/go.mod h1:oDXq+E5IL5q0U8uSBcoAXzTzInwy5lEgC91HoKtbmZk= +github.com/koron/go-ssdp v0.0.6 h1:Jb0h04599eq/CY7rB5YEqPS83HmRfHP2azkxMN2rFtU= +github.com/koron/go-ssdp v0.0.6/go.mod h1:0R9LfRJGek1zWTjN3JUNlm5INCDYGpRDfAptnct63fI= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -576,15 +554,17 @@ github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2 github.com/leanovate/gopter v0.2.11 h1:vRjThO1EKPb/1NsDXuDrzldR28RLkBflWYcU9CvzWu4= github.com/leanovate/gopter v0.2.11/go.mod h1:aK3tzZP/C+p1m3SPRE4SYZFGP7jjkuSI4f7Xvpt0S9c= github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= -github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= -github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/libdns/libdns v0.2.2 h1:O6ws7bAfRPaBsgAYt8MDe2HcNBGC29hkZ9MX2eUSX3s= +github.com/libdns/libdns v0.2.2/go.mod h1:4Bj9+5CQiNMVGf87wjX4CY3HQJypUHRuLvlsfsZqLWQ= github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= -github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= -github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= -github.com/libp2p/go-libp2p v0.33.2 h1:vCdwnFxoGOXMKmaGHlDSnL4bM3fQeW8pgIa9DECnb40= -github.com/libp2p/go-libp2p v0.33.2/go.mod h1:zTeppLuCvUIkT118pFVzA8xzP/p2dJYOMApCkFh0Yww= +github.com/libp2p/go-flow-metrics v0.2.0 h1:EIZzjmeOE6c8Dav0sNv35vhZxATIXWZg6j/C08XmmDw= +github.com/libp2p/go-flow-metrics v0.2.0/go.mod h1:st3qqfu8+pMfh+9Mzqb2GTiwrAGjIPszEjZmtksN8Jc= +github.com/libp2p/go-libp2p v0.46.0 h1:0T2yvIKpZ3DVYCuPOFxPD1layhRU486pj9rSlGWYnDM= +github.com/libp2p/go-libp2p v0.46.0/go.mod h1:TbIDnpDjBLa7isdgYpbxozIVPBTmM/7qKOJP4SFySrQ= github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94= github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8= github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUIK5WDu6iPUA= @@ -592,20 +572,20 @@ github.com/libp2p/go-libp2p-testing v0.12.0/go.mod h1:KcGDRXyN7sQCllucn1cOOS+Dmm github.com/libp2p/go-maddr-filter v0.1.0/go.mod h1:VzZhTXkMucEGGEOSKddrwGiOv0tUhgnKqNEmIAz/bPU= github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0= github.com/libp2p/go-msgio v0.3.0/go.mod h1:nyRM819GmVaF9LX3l03RMh10QdOroF++NBbxAb0mmDM= -github.com/libp2p/go-nat v0.2.0 h1:Tyz+bUFAYqGyJ/ppPPymMGbIgNRH+WqC5QrT5fKrrGk= -github.com/libp2p/go-nat v0.2.0/go.mod h1:3MJr+GRpRkyT65EpVPBstXLvOlAPzUVlG6Pwg9ohLJk= -github.com/libp2p/go-netroute v0.2.1 h1:V8kVrpD8GK0Riv15/7VN6RbUQ3URNZVosw7H2v9tksU= -github.com/libp2p/go-netroute v0.2.1/go.mod h1:hraioZr0fhBjG0ZRXJJ6Zj2IVEVNx6tDTFQfSmcq7mQ= +github.com/libp2p/go-netroute v0.3.0 h1:nqPCXHmeNmgTJnktosJ/sIef9hvwYCrsLxXmfNks/oc= +github.com/libp2p/go-netroute v0.3.0/go.mod h1:Nkd5ShYgSMS5MUKy/MU2T57xFoOKvvLR92Lic48LEyA= github.com/libp2p/go-reuseport v0.4.0 h1:nR5KU7hD0WxXCJbmw7r2rhRYruNRl2koHw8fQscQm2s= github.com/libp2p/go-reuseport v0.4.0/go.mod h1:ZtI03j/wO5hZVDFo2jKywN6bYKWLOy8Se6DrI2E1cLU= -github.com/libp2p/go-yamux/v4 v4.0.1 h1:FfDR4S1wj6Bw2Pqbc8Uz7pCxeRBPbwsBbEdfwiCypkQ= -github.com/libp2p/go-yamux/v4 v4.0.1/go.mod h1:NWjl8ZTLOGlozrXSOZ/HlfG++39iKNnM5wwmtQP1YB4= +github.com/libp2p/go-yamux/v5 v5.0.1 h1:f0WoX/bEF2E8SbE4c/k1Mo+/9z0O4oC/hWEA+nfYRSg= +github.com/libp2p/go-yamux/v5 v5.0.1/go.mod h1:en+3cdX51U0ZslwRdRLrvQsdayFt3TSUKvBGErzpWbU= github.com/lucas-clemente/quic-go v0.15.2/go.mod h1:qxmO5Y4ZMhdNkunGfxuZnZXnJwYpW9vjQkyrZ7BsgUI= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/marcopolo/simnet v0.0.1 h1:rSMslhPz6q9IvJeFWDoMGxMIrlsbXau3NkuIXHGJxfg= +github.com/marcopolo/simnet v0.0.1/go.mod h1:WDaQkgLAjqDUEBAOXz22+1j6wXKfGlC5sD5XWt3ddOs= github.com/marten-seemann/qpack v0.1.0/go.mod h1:LFt1NU/Ptjip0C2CPkhimBz5CGE3WGDAUWqna+CNTrI= github.com/marten-seemann/qtls v0.8.0/go.mod h1:Lao6jDqlCfxyLKYFmZXGm2LSHBgVn+P+ROOex6YkT+k= github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd h1:br0buuQ854V8u83wA0rVZ8ttrq5CpaPZdvrK0LP2lOk= @@ -623,7 +603,6 @@ github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= @@ -633,11 +612,12 @@ github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mholt/acmez/v3 v3.0.0 h1:r1NcjuWR0VaKP2BTjDK9LRFBw/WvURx3jlaEUl9Ht8E= +github.com/mholt/acmez/v3 v3.0.0/go.mod h1:L1wOU06KKvq7tswuMDwKdcHeKpFFgkppZy/y0DFxagQ= github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= -github.com/miekg/dns v1.1.58 h1:ca2Hdkz+cDg/7eNF6V56jjzuZ4aCAE+DbVkILdQWG/4= -github.com/miekg/dns v1.1.58/go.mod h1:Ypv+3b/KadlvW9vJfXOTf300O4UqaHFzFCuHz+rPkBY= +github.com/miekg/dns v1.1.66 h1:FeZXOS3VCVsKnEAd+wBkjMC3D2K+ww66Cq3VnCINuJE= +github.com/miekg/dns v1.1.66/go.mod h1:jGFzBsSNbJw6z1HYut1RKBKHA9PBdxeHrZG8J+gC2WE= github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c h1:bzE/A84HN25pxAuk9Eej1Kz9OUelF97nAc82bDquQI8= github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c/go.mod h1:0SQS9kMwD2VsyFEB++InYyBJroV/FRmBgcydeSUcJms= github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b h1:z78hV3sbSMAUoyUMM0I83AUIT6Hu17AWfgjzIbtrYFc= @@ -678,29 +658,27 @@ github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoR github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo= -github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4= github.com/multiformats/go-multiaddr v0.2.2/go.mod h1:NtfXiOtHvghW9KojvtySjH5y0u0xW5UouOmQQrn6a3Y= github.com/multiformats/go-multiaddr v0.3.2/go.mod h1:lCKNGP1EQ1eZ35Za2wlqnabm9xQkib3fyB+nZXHLag0= -github.com/multiformats/go-multiaddr v0.12.3 h1:hVBXvPRcKG0w80VinQ23P5t7czWgg65BmIvQKjDydU8= -github.com/multiformats/go-multiaddr v0.12.3/go.mod h1:sBXrNzucqkFJhvKOiwwLyqamGa/P5EIXNPLovyhQCII= -github.com/multiformats/go-multiaddr-dns v0.3.1 h1:QgQgR+LQVt3NPTjbrLLpsaT2ufAA2y0Mkk+QRVJbW3A= -github.com/multiformats/go-multiaddr-dns v0.3.1/go.mod h1:G/245BRQ6FJGmryJCrOuTdB37AMA5AMOVuO6NY3JwTk= +github.com/multiformats/go-multiaddr v0.16.1 h1:fgJ0Pitow+wWXzN9do+1b8Pyjmo8m5WhGfzpL82MpCw= +github.com/multiformats/go-multiaddr v0.16.1/go.mod h1:JSVUmXDjsVFiW7RjIFMP7+Ev+h1DTbiJgVeTV/tcmP0= +github.com/multiformats/go-multiaddr-dns v0.4.1 h1:whi/uCLbDS3mSEUMb1MsoT4uzUeZB0N32yzufqS0i5M= +github.com/multiformats/go-multiaddr-dns v0.4.1/go.mod h1:7hfthtB4E4pQwirrz+J0CcDUfbWzTqEzVyYKKIKpgkc= github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E= github.com/multiformats/go-multiaddr-fmt v0.1.0/go.mod h1:hGtDIW4PU4BqJ50gW2quDuPVjyWNZxToGUh/HwTZYJo= github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= -github.com/multiformats/go-multicodec v0.9.0 h1:pb/dlPnzee/Sxv/j4PmkDRxCOi3hXTz3IbPKOXWJkmg= -github.com/multiformats/go-multicodec v0.9.0/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k= +github.com/multiformats/go-multicodec v0.9.1 h1:x/Fuxr7ZuR4jJV4Os5g444F7xC4XmyUaT/FWtE+9Zjo= +github.com/multiformats/go-multicodec v0.9.1/go.mod h1:LLWNMtyV5ithSBUo3vFIMaeDy+h3EbkMTek1m+Fybbo= github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg= github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= -github.com/multiformats/go-multistream v0.5.0 h1:5htLSLl7lvJk3xx3qT/8Zm9J4K8vEOf/QGkvOGQAyiE= -github.com/multiformats/go-multistream v0.5.0/go.mod h1:n6tMZiwiP2wUsR8DgfDWw1dydlEqV3l6N3/GBsX6ILA= -github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +github.com/multiformats/go-multistream v0.6.1 h1:4aoX5v6T+yWmc2raBHsTvzmFhOI8WVOer28DeBBEYdQ= +github.com/multiformats/go-multistream v0.6.1/go.mod h1:ksQf6kqHAb6zIsyw7Zm+gAuVo57Qbq84E27YlYqavqw= github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= @@ -722,20 +700,16 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo/v2 v2.15.0 h1:79HwNRBAZHOEwrczrgSOPy+eFTTlIGELKy5as+ClttY= -github.com/onsi/ginkgo/v2 v2.15.0/go.mod h1:HlxMHtYF57y6Dpf+mc5529KKmSq9h2FpCF+/ZkwUxKM= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= -github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= -github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.2.0 h1:z97+pHb3uELt/yiAWD691HNHQIF07bE7dzrbT927iTk= -github.com/opencontainers/runtime-spec v1.2.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/onsi/gomega v1.36.3 h1:hID7cr8t3Wp26+cYnfcjR6HpJ00fdogN6dqZ1t6IylU= +github.com/onsi/gomega v1.36.3/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.0.3-0.20180606204148-bd9c31933947/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= @@ -756,26 +730,59 @@ github.com/peterh/liner v1.2.1 h1:O4BlKaq/LWu6VRWmol4ByWfzx6MfXc5Op5HETyIy5yg= github.com/peterh/liner v1.2.1/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pion/dtls/v2 v2.2.8 h1:BUroldfiIbV9jSnC6cKOMnyiORRWrWWpV11JUyEu5OA= -github.com/pion/dtls/v2 v2.2.8/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= -github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= +github.com/pion/datachannel v1.5.10 h1:ly0Q26K1i6ZkGf42W7D4hQYR90pZwzFOjTq5AuCKk4o= +github.com/pion/datachannel v1.5.10/go.mod h1:p/jJfC9arb29W7WrxyKbepTU20CFgyx5oLo8Rs4Py/M= +github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= +github.com/pion/dtls/v2 v2.2.12 h1:KP7H5/c1EiVAAKUmXyCzPiQe5+bCJrpOeKg/L05dunk= +github.com/pion/dtls/v2 v2.2.12/go.mod h1:d9SYc9fch0CqK90mRk1dC7AkzzpwJj6u2GU3u+9pqFE= +github.com/pion/dtls/v3 v3.0.6 h1:7Hkd8WhAJNbRgq9RgdNh1aaWlZlGpYTzdqjy9x9sK2E= +github.com/pion/dtls/v3 v3.0.6/go.mod h1:iJxNQ3Uhn1NZWOMWlLxEEHAN5yX7GyPvvKw04v9bzYU= +github.com/pion/ice/v4 v4.0.10 h1:P59w1iauC/wPk9PdY8Vjl4fOFL5B+USq1+xbDcN6gT4= +github.com/pion/ice/v4 v4.0.10/go.mod h1:y3M18aPhIxLlcO/4dn9X8LzLLSma84cx6emMSu14FGw= +github.com/pion/interceptor v0.1.40 h1:e0BjnPcGpr2CFQgKhrQisBU7V3GXK6wrfYrGYaU6Jq4= +github.com/pion/interceptor v0.1.40/go.mod h1:Z6kqH7M/FYirg3frjGJ21VLSRJGBXB/KqaTIrdqnOic= github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms= +github.com/pion/logging v0.2.3 h1:gHuf0zpoh1GW67Nr6Gj4cv5Z9ZscU7g/EaoC/Ke/igI= +github.com/pion/logging v0.2.3/go.mod h1:z8YfknkquMe1csOrxK5kc+5/ZPAzMxbKLX5aXpbpC90= +github.com/pion/mdns/v2 v2.0.7 h1:c9kM8ewCgjslaAmicYMFQIde2H9/lrZpjBkN8VwoVtM= +github.com/pion/mdns/v2 v2.0.7/go.mod h1:vAdSYNAT0Jy3Ru0zl2YiW3Rm/fJCwIeM0nToenfOJKA= +github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= +github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= +github.com/pion/rtcp v1.2.15 h1:LZQi2JbdipLOj4eBjK4wlVoQWfrZbh3Q6eHtWtJBZBo= +github.com/pion/rtcp v1.2.15/go.mod h1:jlGuAjHMEXwMUHK78RgX0UmEJFV4zUKOFHR7OP+D3D0= +github.com/pion/rtp v1.8.19 h1:jhdO/3XhL/aKm/wARFVmvTfq0lC/CvN1xwYKmduly3c= +github.com/pion/rtp v1.8.19/go.mod h1:bAu2UFKScgzyFqvUKmbvzSdPr+NGbZtv6UB2hesqXBk= +github.com/pion/sctp v1.8.39 h1:PJma40vRHa3UTO3C4MyeJDQ+KIobVYRZQZ0Nt7SjQnE= +github.com/pion/sctp v1.8.39/go.mod h1:cNiLdchXra8fHQwmIoqw0MbLLMs+f7uQ+dGMG2gWebE= +github.com/pion/sdp/v3 v3.0.13 h1:uN3SS2b+QDZnWXgdr69SM8KB4EbcnPnPf2Laxhty/l4= +github.com/pion/sdp/v3 v3.0.13/go.mod h1:88GMahN5xnScv1hIMTqLdu/cOcUkj6a9ytbncwMCq2E= +github.com/pion/srtp/v3 v3.0.6 h1:E2gyj1f5X10sB/qILUGIkL4C2CqK269Xq167PbGCc/4= +github.com/pion/srtp/v3 v3.0.6/go.mod h1:BxvziG3v/armJHAaJ87euvkhHqWe9I7iiOy50K2QkhY= github.com/pion/stun v0.6.1 h1:8lp6YejULeHBF8NmV8e2787BogQhduZugh5PdhDyyN4= +github.com/pion/stun v0.6.1/go.mod h1:/hO7APkX4hZKu/D0f2lHzNyvdkTGtIy3NDmLR7kSz/8= github.com/pion/stun/v2 v2.0.0 h1:A5+wXKLAypxQri59+tmQKVs7+l6mMM+3d+eER9ifRU0= github.com/pion/stun/v2 v2.0.0/go.mod h1:22qRSh08fSEttYUmJZGlriq9+03jtVmXNODgLccj8GQ= -github.com/pion/transport/v2 v2.2.4 h1:41JJK6DZQYSeVLxILA2+F4ZkKb4Xd/tFJZRFZQ9QAlo= +github.com/pion/stun/v3 v3.0.0 h1:4h1gwhWLWuZWOJIJR9s2ferRO+W3zA/b6ijOI6mKzUw= +github.com/pion/stun/v3 v3.0.0/go.mod h1:HvCN8txt8mwi4FBvS3EmDghW6aQJ24T+y+1TKjB5jyU= +github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g= github.com/pion/transport/v2 v2.2.4/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= -github.com/pion/transport/v3 v3.0.1 h1:gDTlPJwROfSfz6QfSi0ZmeCSkFcnWWiiR9ES0ouANiM= -github.com/pion/transport/v3 v3.0.1/go.mod h1:UY7kiITrlMv7/IKgd5eTUcaahZx5oUN3l9SzK5f5xE0= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pion/transport/v2 v2.2.10 h1:ucLBLE8nuxiHfvkFKnkDQRYWYfp8ejf4YBOPfaQpw6Q= +github.com/pion/transport/v2 v2.2.10/go.mod h1:sq1kSLWs+cHW9E+2fJP95QudkzbK7wscs8yYgQToO5E= +github.com/pion/transport/v3 v3.0.7 h1:iRbMH05BzSNwhILHoBoAPxoB9xQgOaJk+591KC9P1o0= +github.com/pion/transport/v3 v3.0.7/go.mod h1:YleKiTZ4vqNxVwh77Z0zytYi7rXHl7j6uPLGhhz9rwo= +github.com/pion/turn/v4 v4.0.2 h1:ZqgQ3+MjP32ug30xAbD6Mn+/K4Sxi3SdNOTFf+7mpps= +github.com/pion/turn/v4 v4.0.2/go.mod h1:pMMKP/ieNAG/fN5cZiN4SDuyKsXtNTr0ccN7IToA1zs= +github.com/pion/webrtc/v4 v4.1.2 h1:mpuUo/EJ1zMNKGE79fAdYNFZBX790KE7kQQpLMjjR54= +github.com/pion/webrtc/v4 v4.1.2/go.mod h1:xsCXiNAmMEjIdFxAYU0MbB3RwRieJsegSB2JZsGN+8U= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pkg/term v0.0.0-20180730021639-bffc007b7fd5/go.mod h1:eCbImbZ95eXtAUIbLAuAVnBnwf83mjf6QIVH8SHYwqQ= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -787,14 +794,14 @@ github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqr github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= -github.com/prometheus/client_golang v1.21.1 h1:DOvXXTqVzvkIewV/CDPFdejpMCGeMcbGCQ8YOmu+Ibk= -github.com/prometheus/client_golang v1.21.1/go.mod h1:U9NM32ykUErtVBxdvD3zfi+EuFkkaBvMb09mIfe0Zgg= +github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= +github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= -github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= @@ -806,8 +813,8 @@ github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9 github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.35.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= -github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= -github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= +github.com/prometheus/common v0.64.0 h1:pdZeA+g617P7oGv1CzdTzyeShxAGrTBsolKNOLQPGO4= +github.com/prometheus/common v0.64.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= @@ -818,20 +825,19 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= -github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= -github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -github.com/prometheus/statsd_exporter v0.22.7 h1:7Pji/i2GuhK6Lu7DHrtTkFmNBCudCPT1pX2CziuyQR0= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/prometheus/statsd_exporter v0.22.7/go.mod h1:N/TevpjkIh9ccs6nuzY3jQn9dFqnUakOjnEuMPJJJnI= +github.com/prometheus/statsd_exporter v0.26.1 h1:ucbIAdPmwAUcA+dU+Opok8Qt81Aw8HanlO+2N/Wjv7w= +github.com/prometheus/statsd_exporter v0.26.1/go.mod h1:XlDdjAmRmx3JVvPPYuFNUg+Ynyb5kR69iPPkQjxXFMk= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/prometheus/tsdb v0.10.0/go.mod h1:oi49uRhEe9dPUTlS3JRZOwJuVi6tmh10QSgwXEyGCt4= -github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= -github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= -github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM= -github.com/quic-go/quic-go v0.42.0/go.mod h1:132kz4kL3F9vxhW3CtQJLDVwcFe5wdWeJXXijhsO57M= -github.com/quic-go/webtransport-go v0.6.0 h1:CvNsKqc4W2HljHJnoT+rMmbRJybShZ0YPFDD3NxaZLY= -github.com/quic-go/webtransport-go v0.6.0/go.mod h1:9KjU4AEBqEQidGHNDkZrb8CAa1abRaosM2yGOyiikEc= -github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtBsk= -github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU= +github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= +github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= +github.com/quic-go/quic-go v0.57.1 h1:25KAAR9QR8KZrCZRThWMKVAwGoiHIrNbT72ULHTuI10= +github.com/quic-go/quic-go v0.57.1/go.mod h1:ly4QBAjHA2VhdnxhojRsCUOeJwKYg+taDlos92xb1+s= +github.com/quic-go/webtransport-go v0.9.0 h1:jgys+7/wm6JarGDrW+lD/r9BGqBAmqY/ssklE09bA70= +github.com/quic-go/webtransport-go v0.9.0/go.mod h1:4FUYIiUc75XSsF6HShcLeXXYZJ9AGwo/xh3L8M/P1ao= github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -839,10 +845,8 @@ github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRr github.com/rjeczalik/notify v0.9.2/go.mod h1:aErll2f0sUX9PXZnVNyeiObbmTlk5jnMoCa4QEjJeqM= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= @@ -884,11 +888,7 @@ github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5k github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/assertions v1.1.1 h1:T/YLemO5Yp7KPzS+lVtu+WsHn8yoSwTfItdAd1r3cck= -github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= -github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= @@ -927,8 +927,10 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/stvp/go-udp-testing v0.0.0-20201019212854-469649b16807/go.mod h1:7jxmlfBCDBXRzr0eAQJ48XC1hBu1np4CS5+cHEYfwpc= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= @@ -956,15 +958,13 @@ github.com/uber/jaeger-client-go v2.24.0+incompatible h1:CGchgJcHsDd2jWnaL4XngBy github.com/uber/jaeger-client-go v2.24.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw= github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= -github.com/urfave/cli v1.22.2 h1:gsqYFH8bb9ekPA12kRo0hfjngWQjkJPlN9R0N78BoUo= -github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w= github.com/urfave/cli/v2 v2.27.5/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= -github.com/vmihailenco/msgpack/v5 v5.3.4 h1:qMKAwOV+meBw2Y8k9cVwAy7qErtYCwBzZ2ellBfvnqc= -github.com/vmihailenco/msgpack/v5 v5.3.4/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= +github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8= +github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/wealdtech/go-ens/v3 v3.5.1 h1:0VqkCjIGfIVdwHIf2QqYWWt3bbR1UE7RwBGx7YPpufQ= @@ -974,6 +974,9 @@ github.com/wealdtech/go-multicodec v1.4.0/go.mod h1:aedGMaTeYkIqi/KCPre1ho5rTb3h github.com/wealdtech/go-string2eth v1.1.0 h1:USJQmysUrBYYmZs7d45pMb90hRSyEwizP7lZaOZLDAw= github.com/wealdtech/go-string2eth v1.1.0/go.mod h1:RUzsLjJtbZaJ/3UKn9kY19a/vCCUHtEWoUW3uiK6yGU= github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/wlynxg/anet v0.0.3/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA= +github.com/wlynxg/anet v0.0.5 h1:J3VJGi1gvo0JwZ/P1/Yc/8p63SoW98B5dHkYDmpgvvU= +github.com/wlynxg/anet v0.0.5/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= @@ -982,7 +985,13 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY= +github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/blake3 v0.2.4 h1:KYQPkhpRtcqh0ssGYcKLG1JYvddkEA8QwCM/yBqhaZI= +github.com/zeebo/blake3 v0.2.4/go.mod h1:7eeQ6d2iXWRGF6npfaxl2CU+xy2Fjo2gxeyZGCRUjcE= +github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo= +github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= gitlab.com/nolash/go-mockbytes v0.0.7 h1:9XVFpEfY67kGBVJve3uV19kzqORdlo7V+q09OE6Yo54= gitlab.com/nolash/go-mockbytes v0.0.7/go.mod h1:KKOpNTT39j2Eo+P6uUTOncntfeKY6AFh/2CxuD5MpgE= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= @@ -997,27 +1006,25 @@ go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/dig v1.17.1 h1:Tga8Lz8PcYNsWsyHMZ1Vm0OQOUaJNDyvPImgbAu9YSc= -go.uber.org/dig v1.17.1/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE= -go.uber.org/fx v1.20.1 h1:zVwVQGS8zYvhh9Xxcu4w1M6ESyeMzebzj2NbSayZ4Mk= -go.uber.org/fx v1.20.1/go.mod h1:iSYNbHf2y55acNCwCXKx7LbWb5WG1Bnue5RDXz1OREg= -go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/dig v1.19.0 h1:BACLhebsYdpQ7IROQ1AGPjrXcP5dF80U3gKoFzbaq/4= +go.uber.org/dig v1.19.0/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE= +go.uber.org/fx v1.24.0 h1:wE8mruvpg2kiiL1Vqd0CC+tr0/24XIB10Iwp2lLWzkg= +go.uber.org/fx v1.24.0/go.mod h1:AmDeGyS+ZARGKM4tlH4FY2Jr63VjbEDJHtqXTGP5hbo= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= -go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= +go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= +go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.uber.org/zap/exp v0.3.0 h1:6JYzdifzYkGmTdRR59oYH+Ng7k49H9qVpWwNSsGJj3U= +go.uber.org/zap/exp v0.3.0/go.mod h1:5I384qq7XGxYyByIhHm6jg5CHkGY0nsTfbDLgDDlgJQ= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -1042,7 +1049,10 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1058,8 +1068,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20240213143201-ec583247a57a h1:HinSgX1tJRX3KsL//Gxynpw5CTOAIPhgL4W8PNiIpVE= -golang.org/x/exp v0.0.0-20240213143201-ec583247a57a/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= +golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 h1:bsqhLWFR6G6xiQcb+JoGqdKdRU6WzPWmK8E0jxTjzo4= +golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -1083,6 +1093,8 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1131,12 +1143,16 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1161,9 +1177,10 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= -golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1198,7 +1215,6 @@ golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1220,30 +1236,27 @@ golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/telemetry v0.0.0-20251008203120-078029d740a8 h1:LvzTn0GQhWuvKH/kVRS3R3bVAsdQWI7hvfLHGgh9+lU= @@ -1251,6 +1264,11 @@ golang.org/x/telemetry v0.0.0-20251008203120-078029d740a8/go.mod h1:Pi4ztBfryZoJ golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= +golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1261,6 +1279,10 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1268,8 +1290,8 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= -golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= +golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1322,7 +1344,8 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1427,8 +1450,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= -google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1440,8 +1463,8 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww= -gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/mail.v2 v2.3.1/go.mod h1:htwXN1Qh09vZJ1NVKxQqHPBaCBbzKhp5GzuJEA4VJWw= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= @@ -1476,8 +1499,8 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= -lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI= -lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= +lukechampine.com/blake3 v1.4.1 h1:I3Smz7gso8w4/TunLKec6K2fn+kyKtDxr/xcQEN84Wg= +lukechampine.com/blake3 v1.4.1/go.mod h1:QFosUxmjB8mnrWFSNwKmvxHpfY72bmD2tQ0kBMM3kwo= resenje.org/daemon v0.1.2/go.mod h1:mF5JRpH3EbrxI9WoeKY78e6PqSsbBtX9jAQL5vj/GBA= resenje.org/email v0.1.3/go.mod h1:OhAVLRG3vqd9NSgayN3pAgzxTmc2B6mAefgShZvEgf0= resenje.org/feed v0.1.2 h1:3OianQkoI4EalWx1SlzHtGjUMsoB4XTJQbeehWiyeFI= diff --git a/packaging/bee.yaml b/packaging/bee.yaml index 42ee156d0de..41a6b2a2568 100644 --- a/packaging/bee.yaml +++ b/packaging/bee.yaml @@ -120,3 +120,15 @@ password-file: "/var/lib/bee/password" # welcome-message: "" ## withdrawal target addresses # withdrawal-addresses-whitelist: [] +## enable wss p2p connections (default: false) +# p2p-wss-enable: false +## wss address (default: :1635) +# p2p-wss-addr: :1635 +## WSS NAT exposed address +# nat-wss-addr: "" +## autotls domain (default: libp2p.direct) +# autotls-domain: "" +## autotls registration endpoint (default: https://registration.libp2p.direct) +# autotls-registration-endpoint: "" +## autotls ca endpoint (default: https://acme-v02.api.letsencrypt.org/directory) +# autotls-ca-endpoint: "" diff --git a/packaging/homebrew-amd64/bee.yaml b/packaging/homebrew-amd64/bee.yaml index 14a32f70116..4e93b8e3bab 100644 --- a/packaging/homebrew-amd64/bee.yaml +++ b/packaging/homebrew-amd64/bee.yaml @@ -120,3 +120,15 @@ password-file: "/usr/local/var/lib/swarm-bee/password" # welcome-message: "" ## withdrawal target addresses # withdrawal-addresses-whitelist: [] +## enable wss p2p connections (default: false) +# p2p-wss-enable: false +## wss address (default: :1635) +# p2p-wss-addr: :1635 +## WSS NAT exposed address +# nat-wss-addr: "" +## autotls domain (default: libp2p.direct) +# autotls-domain: "" +## autotls registration endpoint (default: https://registration.libp2p.direct) +# autotls-registration-endpoint: "" +## autotls ca endpoint (default: https://acme-v02.api.letsencrypt.org/directory) +# autotls-ca-endpoint: "" diff --git a/packaging/homebrew-arm64/bee.yaml b/packaging/homebrew-arm64/bee.yaml index 55d522b601c..291288c3eed 100644 --- a/packaging/homebrew-arm64/bee.yaml +++ b/packaging/homebrew-arm64/bee.yaml @@ -120,3 +120,15 @@ password-file: "/opt/homebrew/var/lib/swarm-bee/password" # welcome-message: "" ## withdrawal target addresses # withdrawal-addresses-whitelist: [] +## enable wss p2p connections (default: false) +# p2p-wss-enable: false +## wss address (default: :1635) +# p2p-wss-addr: :1635 +## WSS NAT exposed address +# nat-wss-addr: "" +## autotls domain (default: libp2p.direct) +# autotls-domain: "" +## autotls registration endpoint (default: https://registration.libp2p.direct) +# autotls-registration-endpoint: "" +## autotls ca endpoint (default: https://acme-v02.api.letsencrypt.org/directory) +# autotls-ca-endpoint: "" diff --git a/packaging/scoop/bee.yaml b/packaging/scoop/bee.yaml index 70bed38f947..d73c9f4feb1 100644 --- a/packaging/scoop/bee.yaml +++ b/packaging/scoop/bee.yaml @@ -120,3 +120,15 @@ password-file: "./password" # welcome-message: "" ## withdrawal target addresses # withdrawal-addresses-whitelist: [] +## enable wss p2p connections (default: false) +# p2p-wss-enable: false +## wss address (default: :1635) +# p2p-wss-addr: :1635 +## WSS NAT exposed address +# nat-wss-addr: "" +## autotls domain (default: libp2p.direct) +# autotls-domain: "" +## autotls registration endpoint (default: https://registration.libp2p.direct) +# autotls-registration-endpoint: "" +## autotls ca endpoint (default: https://acme-v02.api.letsencrypt.org/directory) +# autotls-ca-endpoint: "" diff --git a/pkg/api/peer_test.go b/pkg/api/peer_test.go index 15694632709..1175e7d6bc4 100644 --- a/pkg/api/peer_test.go +++ b/pkg/api/peer_test.go @@ -62,7 +62,6 @@ func TestConnect(t *testing.T) { }) t.Run("ok", func(t *testing.T) { - t.Parallel() jsonhttptest.Request(t, testServer, http.MethodPost, "/connect"+underlay, http.StatusOK, jsonhttptest.WithExpectedJSONResponse(api.PeerConnectResponse{ Address: overlay.String(), @@ -71,7 +70,6 @@ func TestConnect(t *testing.T) { }) t.Run("error", func(t *testing.T) { - t.Parallel() jsonhttptest.Request(t, testServer, http.MethodPost, "/connect"+errorUnderlay, http.StatusInternalServerError, jsonhttptest.WithExpectedJSONResponse(jsonhttp.StatusResponse{ Code: http.StatusInternalServerError, @@ -81,7 +79,6 @@ func TestConnect(t *testing.T) { }) t.Run("error - add peer", func(t *testing.T) { - t.Parallel() testServer, _, _, _ := newTestServer(t, testServerOptions{ P2P: mock.New(mock.WithConnectFunc(func(ctx context.Context, addrs []ma.Multiaddr) (*bzz.Address, error) { for _, addr := range addrs { diff --git a/pkg/api/util.go b/pkg/api/util.go index 60780c5acb4..d8911f4ecf4 100644 --- a/pkg/api/util.go +++ b/pkg/api/util.go @@ -210,14 +210,23 @@ func mapStructure(input, output any, hooks map[string]func(v string) (string, er case reflect.String: field.SetString(value) case reflect.Slice: - if value == "" { - return nil // Nil slice. - } - val, err := hex.DecodeString(value) - if err != nil { - return err + switch field.Type() { + case reflect.TypeFor[multiaddr.Multiaddr](): + val, err := multiaddr.NewMultiaddr(value) + if err != nil { + return err + } + field.Set(reflect.ValueOf(val)) + default: + if value == "" { + return nil // Nil slice. + } + val, err := hex.DecodeString(value) + if err != nil { + return err + } + field.SetBytes(val) } - field.SetBytes(val) case reflect.Array: switch field.Interface().(type) { case common.Hash: @@ -251,15 +260,6 @@ func mapStructure(input, output any, hooks map[string]func(v string) (string, er } field.Set(reflect.ValueOf(*val)) } - case reflect.Interface: - switch field.Type() { - case reflect.TypeFor[multiaddr.Multiaddr](): - val, err := multiaddr.NewMultiaddr(value) - if err != nil { - return err - } - field.Set(reflect.ValueOf(val)) - } default: return fmt.Errorf("unsupported type %T", field.Interface()) } diff --git a/pkg/api/util_test.go b/pkg/api/util_test.go index d9ae6732810..b35280de4bd 100644 --- a/pkg/api/util_test.go +++ b/pkg/api/util_test.go @@ -17,6 +17,7 @@ import ( "github.com/ethersphere/bee/v2/pkg/api" "github.com/ethersphere/bee/v2/pkg/swarm" "github.com/google/go-cmp/cmp" + "github.com/multiformats/go-multiaddr" ) type ( @@ -95,11 +96,20 @@ type ( mapSwarmAddressTest struct { SwarmAddressVal swarm.Address `map:"swarmAddressVal"` } + + mapMultiaddrTest struct { + MultiaddrVal multiaddr.Multiaddr `map:"multiaddrVal"` + } ) func TestMapStructure(t *testing.T) { t.Parallel() + validMultiaddr, err := multiaddr.NewMultiaddr("/ip4/127.0.0.1/tcp/8080") + if err != nil { + t.Fatal(err) + } + tests := []struct { name string src any @@ -497,6 +507,10 @@ func TestMapStructure(t *testing.T) { name: "swarm.Address value", src: map[string]string{"swarmAddressVal": "1234567890abcdef"}, want: &mapSwarmAddressTest{SwarmAddressVal: swarm.MustParseHexAddress("1234567890abcdef")}, + }, { + name: "multiaddr.Multiaddr value", + src: map[string]string{"multiaddrVal": "/ip4/127.0.0.1/tcp/8080"}, + want: &mapMultiaddrTest{MultiaddrVal: validMultiaddr}, }} for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { diff --git a/pkg/bzz/underlay_test.go b/pkg/bzz/underlay_test.go index f6e4fc7933b..5bc4e34c19d 100644 --- a/pkg/bzz/underlay_test.go +++ b/pkg/bzz/underlay_test.go @@ -106,6 +106,35 @@ func TestDeserializeUnderlays(t *testing.T) { } }) + t.Run("serialize deserialize empty list", func(t *testing.T) { + addrs := []multiaddr.Multiaddr{} + serialized := bzz.SerializeUnderlays(addrs) + if !bytes.Equal(serialized, []byte{bzz.UnderlayListPrefix}) { + t.Errorf("expected %v, got %v", []byte{bzz.UnderlayListPrefix}, serialized) + } + deserialized, err := bzz.DeserializeUnderlays(serialized) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(deserialized) != 0 { + t.Errorf("expected empty slice, got %v", deserialized) + } + }) + + t.Run("serialize deserialize nil list", func(t *testing.T) { + serialized := bzz.SerializeUnderlays(nil) + if !bytes.Equal(serialized, []byte{bzz.UnderlayListPrefix}) { + t.Errorf("expected %v, got %v", []byte{bzz.UnderlayListPrefix}, serialized) + } + deserialized, err := bzz.DeserializeUnderlays(serialized) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(deserialized) != 0 { + t.Errorf("expected empty slice, got %v", deserialized) + } + }) + t.Run("corrupted list - length too long", func(t *testing.T) { maBytes := ip4TCPAddr.Bytes() var buf bytes.Buffer diff --git a/pkg/hive/hive.go b/pkg/hive/hive.go index f5fe1b3bf53..1f49fe2808c 100644 --- a/pkg/hive/hive.go +++ b/pkg/hive/hive.go @@ -12,7 +12,6 @@ package hive import ( "context" - "encoding/hex" "errors" "fmt" "sync" @@ -36,13 +35,11 @@ import ( const loggerName = "hive" const ( - protocolName = "hive" - protocolVersion = "1.1.0" - peersStreamName = "peers" - messageTimeout = 1 * time.Minute // maximum allowed time for a message to be read or written. - maxBatchSize = 30 - pingTimeout = time.Second * 15 // time to wait for ping to succeed - batchValidationTimeout = 5 * time.Minute // prevent lock contention on peer validation + protocolName = "hive" + protocolVersion = "1.1.0" + peersStreamName = "peers" + messageTimeout = 1 * time.Minute // maximum allowed time for a message to be read or written. + maxBatchSize = 30 ) var ( @@ -53,7 +50,7 @@ var ( ) type Service struct { - streamer p2p.StreamerPinger + streamer p2p.Streamer addressBook addressbook.GetPutter addPeersHandler func(...swarm.Address) networkID uint64 @@ -67,9 +64,10 @@ type Service struct { sem *semaphore.Weighted bootnode bool allowPrivateCIDRs bool + overlay swarm.Address } -func New(streamer p2p.StreamerPinger, addressbook addressbook.GetPutter, networkID uint64, bootnode bool, allowPrivateCIDRs bool, logger log.Logger) *Service { +func New(streamer p2p.Streamer, addressbook addressbook.GetPutter, networkID uint64, bootnode bool, allowPrivateCIDRs bool, overlay swarm.Address, logger log.Logger) *Service { svc := &Service{ streamer: streamer, logger: logger.WithName(loggerName).Register(), @@ -83,6 +81,7 @@ func New(streamer p2p.StreamerPinger, addressbook addressbook.GetPutter, network sem: semaphore.NewWeighted(int64(swarm.MaxBins)), bootnode: bootnode, allowPrivateCIDRs: allowPrivateCIDRs, + overlay: overlay, } if !bootnode { @@ -177,6 +176,11 @@ func (s *Service) sendPeers(ctx context.Context, peer swarm.Address, peers []swa w, _ := protobuf.NewWriterAndReader(stream) var peersRequest pb.Peers for _, p := range peers { + if p.Equal(s.overlay) { + s.logger.Debug("skipping self-address in broadcast", "overlay", p.String()) + continue + } + addr, err := s.addressBook.Get(p) if err != nil { if errors.Is(err, addressbook.ErrNotFound) { @@ -263,83 +267,15 @@ func (s *Service) startCheckPeersHandler() { return case newPeers := <-s.peersChan: s.wg.Go(func() { - cctx, cancel := context.WithTimeout(ctx, batchValidationTimeout) - defer cancel() - s.checkAndAddPeers(cctx, newPeers) + s.checkAndAddPeers(newPeers) }) } } }) } -func (s *Service) checkAndAddPeers(ctx context.Context, peers pb.Peers) { - var peersToAdd []swarm.Address - mtx := sync.Mutex{} - wg := sync.WaitGroup{} - - addPeer := func(newPeer *pb.BzzAddress, multiUnderlay []ma.Multiaddr) { - err := s.sem.Acquire(ctx, 1) - if err != nil { - return - } - - wg.Add(1) - go func() { - s.metrics.PeerConnectAttempts.Inc() - - defer func() { - s.sem.Release(1) - wg.Done() - }() - - ctx, cancel := context.WithTimeout(ctx, pingTimeout) - defer cancel() - - var ( - pingSuccessful bool - start time.Time - ) - for _, underlay := range multiUnderlay { - // ping each underlay address, pick first available - start = time.Now() - if _, err := s.streamer.Ping(ctx, underlay); err != nil { - s.logger.Debug("unreachable peer underlay", "peer_address", hex.EncodeToString(newPeer.Overlay), "underlay", underlay, "error", err) - continue - } - pingSuccessful = true - s.logger.Debug("reachable peer underlay", "peer_address", hex.EncodeToString(newPeer.Overlay), "underlay", underlay) - break - } - - if !pingSuccessful { - // none of underlay addresses is available - s.metrics.PingFailureTime.Observe(time.Since(start).Seconds()) - s.metrics.UnreachablePeers.Inc() - return - } - - s.metrics.PingTime.Observe(time.Since(start).Seconds()) - s.metrics.ReachablePeers.Inc() - - bzzAddress := bzz.Address{ - Overlay: swarm.NewAddress(newPeer.Overlay), - Underlays: multiUnderlay, - Signature: newPeer.Signature, - Nonce: newPeer.Nonce, - } - - err := s.addressBook.Put(bzzAddress.Overlay, bzzAddress) - if err != nil { - s.metrics.StorePeerErr.Inc() - s.logger.Warning("skipping peer in response", "peer_address", newPeer.String(), "error", err) - return - } - - mtx.Lock() - peersToAdd = append(peersToAdd, bzzAddress.Overlay) - mtx.Unlock() - }() - } +func (s *Service) checkAndAddPeers(peers pb.Peers) { + peersToAdd := make([]swarm.Address, 0, len(peers.Peers)) for _, p := range peers.Peers { multiUnderlays, err := bzz.DeserializeUnderlays(p.Underlay) @@ -361,10 +297,28 @@ func (s *Service) checkAndAddPeers(ctx context.Context, peers pb.Peers) { continue } - // add peer does not exist in the addressbook - addPeer(p, multiUnderlays) + overlayAddr := swarm.NewAddress(p.Overlay) + + if overlayAddr.Equal(s.overlay) { + s.logger.Debug("skipping self-address in peer list", "overlay", overlayAddr.String()) + continue + } + + bzzAddress := bzz.Address{ + Overlay: overlayAddr, + Underlays: multiUnderlays, + Signature: p.Signature, + Nonce: p.Nonce, + } + + if err := s.addressBook.Put(bzzAddress.Overlay, bzzAddress); err != nil { + s.metrics.StorePeerErr.Inc() + s.logger.Warning("put peer in addressbook", "peer_address", p.String(), "error", err) + continue + } + + peersToAdd = append(peersToAdd, bzzAddress.Overlay) } - wg.Wait() if s.addPeersHandler != nil && len(peersToAdd) > 0 { s.addPeersHandler(peersToAdd...) diff --git a/pkg/hive/hive_test.go b/pkg/hive/hive_test.go index 1f0d6eecd53..ec603c8bb2b 100644 --- a/pkg/hive/hive_test.go +++ b/pkg/hive/hive_test.go @@ -8,7 +8,6 @@ import ( "bytes" "context" "encoding/hex" - "errors" "fmt" "strconv" "testing" @@ -48,13 +47,12 @@ func TestHandlerRateLimit(t *testing.T) { addressbookclean := ab.New(mock.NewStateStore()) - // new recorder for handling Ping + // new recorder streamer := streamtest.New() // create a hive server that handles the incoming stream - server := hive.New(streamer, addressbookclean, networkID, false, true, logger) - testutil.CleanupCloser(t, server) - serverAddress := swarm.RandAddress(t) + server := hive.New(streamer, addressbookclean, networkID, false, true, serverAddress, logger) + testutil.CleanupCloser(t, server) // setup the stream recorder to record stream data serverRecorder := streamtest.New( @@ -95,7 +93,8 @@ func TestHandlerRateLimit(t *testing.T) { } // create a hive client that will do broadcast - client := hive.New(serverRecorder, addressbook, networkID, false, true, logger) + clientAddress := swarm.RandAddress(t) + client := hive.New(serverRecorder, addressbook, networkID, false, true, clientAddress, logger) err := client.BroadcastPeers(context.Background(), serverAddress, peers...) if err != nil { t.Fatal(err) @@ -192,7 +191,6 @@ func TestBroadcastPeers(t *testing.T) { wantOverlays []swarm.Address wantBzzAddresses []bzz.Address allowPrivateCIDRs bool - pingErr func(addr ma.Multiaddr) (time.Duration, error) }{ "OK - single record": { addresee: swarm.MustParseHexAddress("ca1e9f3938cc1425c6061b96ad9eb93e134dfe8734ad490164ef20af9d1cf59c"), @@ -234,24 +232,6 @@ func TestBroadcastPeers(t *testing.T) { wantBzzAddresses: bzzAddresses[:2*hive.MaxBatchSize], allowPrivateCIDRs: true, }, - "OK - single batch - skip ping failures": { - addresee: swarm.MustParseHexAddress("ca1e9f3938cc1425c6061b96ad9eb93e134dfe8734ad490164ef20af9d1cf59c"), - peers: overlays[:15], - wantMsgs: []pb.Peers{{Peers: wantMsgs[0].Peers[:15]}}, - wantOverlays: overlays[:10], - wantBzzAddresses: bzzAddresses[:10], - allowPrivateCIDRs: true, - pingErr: func(addr ma.Multiaddr) (rtt time.Duration, err error) { - for _, v := range bzzAddresses[10:15] { - for _, underlay := range v.Underlays { - if underlay.Equal(addr) { - return rtt, errors.New("ping failure") - } - } - } - return rtt, nil - }, - }, "Ok - don't advertise private CIDRs only": { addresee: overlays[len(overlays)-1], peers: overlays[:15], @@ -291,15 +271,10 @@ func TestBroadcastPeers(t *testing.T) { addressbookclean := ab.New(mock.NewStateStore()) - // new recorder for handling Ping - var streamer *streamtest.Recorder - if tc.pingErr != nil { - streamer = streamtest.New(streamtest.WithPingErr(tc.pingErr)) - } else { - streamer = streamtest.New() - } + streamer := streamtest.New() // create a hive server that handles the incoming stream - server := hive.New(streamer, addressbookclean, networkID, false, true, logger) + serverAddress := swarm.RandAddress(t) + server := hive.New(streamer, addressbookclean, networkID, false, true, serverAddress, logger) testutil.CleanupCloser(t, server) // setup the stream recorder to record stream data @@ -308,7 +283,8 @@ func TestBroadcastPeers(t *testing.T) { ) // create a hive client that will do broadcast - client := hive.New(recorder, addressbook, networkID, false, tc.allowPrivateCIDRs, logger) + clientAddress := swarm.RandAddress(t) + client := hive.New(recorder, addressbook, networkID, false, tc.allowPrivateCIDRs, clientAddress, logger) if err := client.BroadcastPeers(context.Background(), tc.addresee, tc.peers...); err != nil { t.Fatal(err) @@ -470,3 +446,222 @@ func shortHex(b []byte) string { } return s } + +// TestBroadcastPeersSkipsSelf verifies that hive does not broadcast self address +// to other peers, preventing self-connection attempts. +func TestBroadcastPeersSkipsSelf(t *testing.T) { + t.Parallel() + + logger := log.Noop + statestore := mock.NewStateStore() + addressbook := ab.New(statestore) + networkID := uint64(1) + addressbookclean := ab.New(mock.NewStateStore()) + + // Create addresses + serverAddress := swarm.RandAddress(t) + clientAddress := swarm.RandAddress(t) + + // Create a peer address + peer1 := swarm.RandAddress(t) + underlay1, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/1234") + if err != nil { + t.Fatal(err) + } + pk, err := crypto.GenerateSecp256k1Key() + if err != nil { + t.Fatal(err) + } + signer := crypto.NewDefaultSigner(pk) + overlay1, err := crypto.NewOverlayAddress(pk.PublicKey, networkID, block) + if err != nil { + t.Fatal(err) + } + bzzAddr1, err := bzz.NewAddress(signer, []ma.Multiaddr{underlay1}, overlay1, networkID, nonce) + if err != nil { + t.Fatal(err) + } + if err := addressbook.Put(bzzAddr1.Overlay, *bzzAddr1); err != nil { + t.Fatal(err) + } + + // Create self address entry in addressbook + underlayClient, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/9999") + if err != nil { + t.Fatal(err) + } + pkClient, err := crypto.GenerateSecp256k1Key() + if err != nil { + t.Fatal(err) + } + signerClient := crypto.NewDefaultSigner(pkClient) + bzzAddrClient, err := bzz.NewAddress(signerClient, []ma.Multiaddr{underlayClient}, clientAddress, networkID, nonce) + if err != nil { + t.Fatal(err) + } + if err := addressbook.Put(clientAddress, *bzzAddrClient); err != nil { + t.Fatal(err) + } + + // Setup server + streamer := streamtest.New() + server := hive.New(streamer, addressbookclean, networkID, false, true, serverAddress, logger) + testutil.CleanupCloser(t, server) + + serverRecorder := streamtest.New( + streamtest.WithProtocols(server.Protocol()), + ) + + // Setup client + client := hive.New(serverRecorder, addressbook, networkID, false, true, clientAddress, logger) + testutil.CleanupCloser(t, client) + + // Try to broadcast: peer1, clientAddress (self), and another peer + peersIncludingSelf := []swarm.Address{bzzAddr1.Overlay, clientAddress, peer1} + + err = client.BroadcastPeers(context.Background(), serverAddress, peersIncludingSelf...) + if err != nil { + t.Fatal(err) + } + + // Get records + records, err := serverRecorder.Records(serverAddress, "hive", "1.1.0", "peers") + if err != nil { + t.Fatal(err) + } + + if len(records) == 0 { + t.Fatal("expected at least one record") + } + + // Read the messages + messages, err := readAndAssertPeersMsgs(records[0].In(), 1) + if err != nil { + t.Fatal(err) + } + + // Verify that clientAddress (self) was NOT included in broadcast + for _, peerMsg := range messages[0].Peers { + receivedOverlay := swarm.NewAddress(peerMsg.Overlay) + if receivedOverlay.Equal(clientAddress) { + t.Fatal("self address should not be broadcast to peers") + } + } + + // Verify server addressbook eventually contains only the valid peers, not self + err = spinlock.Wait(spinTimeout, func() bool { + overlays, err := addressbookclean.Overlays() + if err != nil { + t.Fatal(err) + } + // Should only have bzzAddr1, not clientAddress + for _, o := range overlays { + if o.Equal(clientAddress) { + return false // self should not be in addressbook + } + } + return true + }) + if err != nil { + t.Fatal("self address found in server addressbook") + } +} + +// TestReceivePeersSkipsSelf verifies that hive does not add self address +// when receiving peer lists from other peers. +func TestReceivePeersSkipsSelf(t *testing.T) { + t.Parallel() + + logger := log.Noop + statestore := mock.NewStateStore() + addressbook := ab.New(statestore) + networkID := uint64(1) + addressbookclean := ab.New(mock.NewStateStore()) + + // Create addresses + serverAddress := swarm.RandAddress(t) + clientAddress := swarm.RandAddress(t) + + // Create a valid peer + underlay1, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/1234") + if err != nil { + t.Fatal(err) + } + pk1, err := crypto.GenerateSecp256k1Key() + if err != nil { + t.Fatal(err) + } + signer1 := crypto.NewDefaultSigner(pk1) + overlay1, err := crypto.NewOverlayAddress(pk1.PublicKey, networkID, block) + if err != nil { + t.Fatal(err) + } + bzzAddr1, err := bzz.NewAddress(signer1, []ma.Multiaddr{underlay1}, overlay1, networkID, nonce) + if err != nil { + t.Fatal(err) + } + if err := addressbook.Put(bzzAddr1.Overlay, *bzzAddr1); err != nil { + t.Fatal(err) + } + + // Create self address entry (serverAddress) that will be sent by client + underlayServer, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/8888") + if err != nil { + t.Fatal(err) + } + pkServer, err := crypto.GenerateSecp256k1Key() + if err != nil { + t.Fatal(err) + } + signerServer := crypto.NewDefaultSigner(pkServer) + bzzAddrServer, err := bzz.NewAddress(signerServer, []ma.Multiaddr{underlayServer}, serverAddress, networkID, nonce) + if err != nil { + t.Fatal(err) + } + // Add server's own address to client's addressbook (so client can send it) + if err := addressbook.Put(serverAddress, *bzzAddrServer); err != nil { + t.Fatal(err) + } + + // Setup server that will receive peers including its own address + streamer := streamtest.New() + server := hive.New(streamer, addressbookclean, networkID, false, true, serverAddress, logger) + testutil.CleanupCloser(t, server) + + serverRecorder := streamtest.New( + streamtest.WithProtocols(server.Protocol()), + ) + + // Setup client + client := hive.New(serverRecorder, addressbook, networkID, false, true, clientAddress, logger) + testutil.CleanupCloser(t, client) + + // Client broadcasts: valid peer and server's own address + peersIncludingSelf := []swarm.Address{bzzAddr1.Overlay, serverAddress} + + err = client.BroadcastPeers(context.Background(), serverAddress, peersIncludingSelf...) + if err != nil { + t.Fatal(err) + } + + // Wait a bit for server to process + time.Sleep(100 * time.Millisecond) + + // Verify server's addressbook does NOT contain its own address + overlays, err := addressbookclean.Overlays() + if err != nil { + t.Fatal(err) + } + + for _, o := range overlays { + if o.Equal(serverAddress) { + t.Fatal("server should not add its own address to addressbook when received from peer") + } + } + + // Verify server does have the valid peer + _, err = addressbookclean.Get(bzzAddr1.Overlay) + if err != nil { + t.Fatalf("expected server to have valid peer in addressbook, got error: %v", err) + } +} diff --git a/pkg/node/bootstrap.go b/pkg/node/bootstrap.go index 047c306aaf2..eab32166ec9 100644 --- a/pkg/node/bootstrap.go +++ b/pkg/node/bootstrap.go @@ -97,12 +97,19 @@ func bootstrapNode( }() p2ps, err := libp2p.New(p2pCtx, signer, networkID, swarmAddress, addr, addressbook, stateStore, lightNodes, logger, tracer, libp2p.Options{ - PrivateKey: libp2pPrivateKey, - NATAddr: o.NATAddr, - EnableWS: o.EnableWS, - WelcomeMessage: o.WelcomeMessage, - FullNode: false, - Nonce: nonce, + PrivateKey: libp2pPrivateKey, + NATAddr: o.NATAddr, + NATWSSAddr: o.NATWSSAddr, + EnableWS: o.EnableWS, + EnableWSS: o.EnableWSS, + WSSAddr: o.WSSAddr, + AutoTLSStorageDir: o.AutoTLSStorageDir, + AutoTLSDomain: o.AutoTLSDomain, + AutoTLSRegistrationEndpoint: o.AutoTLSRegistrationEndpoint, + AutoTLSCAEndpoint: o.AutoTLSCAEndpoint, + WelcomeMessage: o.WelcomeMessage, + FullNode: false, + Nonce: nonce, }) if err != nil { return nil, fmt.Errorf("p2p service: %w", err) @@ -110,7 +117,7 @@ func bootstrapNode( b.p2pService = p2ps b.p2pHalter = p2ps - hive := hive.New(p2ps, addressbook, networkID, o.BootnodeMode, o.AllowPrivateCIDRs, logger) + hive := hive.New(p2ps, addressbook, networkID, o.BootnodeMode, o.AllowPrivateCIDRs, swarmAddress, logger) if err = p2ps.AddProtocol(hive.Protocol()); err != nil { return nil, fmt.Errorf("hive service: %w", err) diff --git a/pkg/node/node.go b/pkg/node/node.go index f20468df338..56cdbc4572f 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -129,12 +129,16 @@ type Options struct { Addr string AllowPrivateCIDRs bool APIAddr string + EnableWSS bool + WSSAddr string + AutoTLSStorageDir string BlockchainRpcEndpoint string BlockProfile bool BlockTime time.Duration BootnodeMode bool Bootnodes []string CacheCapacity uint64 + AutoTLSCAEndpoint string ChainID int64 ChequebookEnable bool CORSAllowedOrigins []string @@ -145,12 +149,15 @@ type Options struct { DBWriteBufferSize uint64 EnableStorageIncentives bool EnableWS bool + AutoTLSDomain string + AutoTLSRegistrationEndpoint string FullNodeMode bool Logger log.Logger MinimumGasTipCap uint64 MinimumStorageRadius uint MutexProfile bool NATAddr string + NATWSSAddr string NeighborhoodSuggester string PaymentEarly int64 PaymentThreshold string @@ -228,6 +235,10 @@ func NewBee( return nil, fmt.Errorf("invalid NAT address %s: %w", o.NATAddr, err) } + if err := validatePublicAddress(o.NATWSSAddr); err != nil { + return nil, fmt.Errorf("invalid NAT WSS address %s: %w", o.NATWSSAddr, err) + } + ctx, ctxCancel := context.WithCancel(ctx) defer func() { // if there's been an error on this function @@ -653,14 +664,21 @@ func NewBee( } p2ps, err := libp2p.New(ctx, signer, networkID, swarmAddress, addr, addressbook, stateStore, lightNodes, logger, tracer, libp2p.Options{ - PrivateKey: libp2pPrivateKey, - NATAddr: o.NATAddr, - EnableWS: o.EnableWS, - WelcomeMessage: o.WelcomeMessage, - FullNode: o.FullNodeMode, - Nonce: nonce, - ValidateOverlay: chainEnabled, - Registry: registry, + PrivateKey: libp2pPrivateKey, + NATAddr: o.NATAddr, + NATWSSAddr: o.NATWSSAddr, + EnableWS: o.EnableWS, + EnableWSS: o.EnableWSS, + WSSAddr: o.WSSAddr, + AutoTLSStorageDir: o.AutoTLSStorageDir, + AutoTLSDomain: o.AutoTLSDomain, + AutoTLSRegistrationEndpoint: o.AutoTLSRegistrationEndpoint, + AutoTLSCAEndpoint: o.AutoTLSCAEndpoint, + WelcomeMessage: o.WelcomeMessage, + FullNode: o.FullNodeMode, + Nonce: nonce, + ValidateOverlay: chainEnabled, + Registry: registry, }) if err != nil { return nil, fmt.Errorf("p2p service: %w", err) @@ -733,7 +751,7 @@ func NewBee( return nil, fmt.Errorf("pingpong service: %w", err) } - hive := hive.New(p2ps, addressbook, networkID, o.BootnodeMode, o.AllowPrivateCIDRs, logger) + hive := hive.New(p2ps, addressbook, networkID, o.BootnodeMode, o.AllowPrivateCIDRs, swarmAddress, logger) if err = p2ps.AddProtocol(hive.Protocol()); err != nil { return nil, fmt.Errorf("hive service: %w", err) diff --git a/pkg/p2p/libp2p/composite_resolver_test.go b/pkg/p2p/libp2p/composite_resolver_test.go new file mode 100644 index 00000000000..43a87319807 --- /dev/null +++ b/pkg/p2p/libp2p/composite_resolver_test.go @@ -0,0 +1,162 @@ +// Copyright 2025 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package libp2p_test + +import ( + "errors" + "testing" + + "github.com/ethersphere/bee/v2/pkg/p2p/libp2p" + "github.com/ethersphere/bee/v2/pkg/p2p/libp2p/internal/handshake" + ma "github.com/multiformats/go-multiaddr" +) + +// mockResolver is a mock implementation of handshake.AdvertisableAddressResolver. +type mockResolver struct { + resolveFunc func(ma.Multiaddr) (ma.Multiaddr, error) + called bool +} + +func (m *mockResolver) Resolve(addr ma.Multiaddr) (ma.Multiaddr, error) { + m.called = true + if m.resolveFunc != nil { + return m.resolveFunc(addr) + } + return addr, nil +} + +func TestCompositeAddressResolver(t *testing.T) { + t.Parallel() + + resolvedAddr, _ := ma.NewMultiaddr("/ip4/1.1.1.1/tcp/1634") + + tests := []struct { + name string + addr string + setup func() (handshake.AdvertisableAddressResolver, *mockResolver, *mockResolver) + wantTCPCalled bool + wantWSSCalled bool + wantErr error + wantResolved bool + }{ + { + name: "wss address", + addr: "/ip4/10.233.99.40/tcp/1635/tls/sni/*.libp2p.direct/ws/p2p/QmWbXocGMpfa8zApx9kCNwfmc35bbRJv136bdtuQjbR4wL", + setup: func() (handshake.AdvertisableAddressResolver, *mockResolver, *mockResolver) { + tcpR := &mockResolver{} + wssR := &mockResolver{ + resolveFunc: func(addr ma.Multiaddr) (ma.Multiaddr, error) { + return resolvedAddr, nil + }, + } + return libp2p.NewCompositeAddressResolver(tcpR, wssR), tcpR, wssR + }, + wantWSSCalled: true, + wantResolved: true, + }, + { + name: "tcp address", + addr: "/ip4/10.233.99.40/tcp/1634/p2p/QmWbXocGMpfa8zApx9kCNwfmc35bbRJv136bdtuQjbR4wL", + setup: func() (handshake.AdvertisableAddressResolver, *mockResolver, *mockResolver) { + tcpR := &mockResolver{ + resolveFunc: func(addr ma.Multiaddr) (ma.Multiaddr, error) { + return resolvedAddr, nil + }, + } + wssR := &mockResolver{} + return libp2p.NewCompositeAddressResolver(tcpR, wssR), tcpR, wssR + }, + wantTCPCalled: true, + wantResolved: true, + }, + { + name: "deprecated wss address", + addr: "/ip4/10.233.99.40/tcp/1635/wss/p2p/QmWbXocGMpfa8zApx9kCNwfmc35bbRJv136bdtuQjbR4wL", + setup: func() (handshake.AdvertisableAddressResolver, *mockResolver, *mockResolver) { + tcpR := &mockResolver{ + resolveFunc: func(addr ma.Multiaddr) (ma.Multiaddr, error) { + return resolvedAddr, nil + }, + } + wssR := &mockResolver{} + return libp2p.NewCompositeAddressResolver(tcpR, wssR), tcpR, wssR + }, + wantTCPCalled: true, + wantResolved: true, + }, + { + name: "ws without tls", + addr: "/ip4/10.233.99.40/tcp/1635/ws/p2p/QmWbXocGMpfa8zApx9kCNwfmc35bbRJv136bdtuQjbR4wL", + setup: func() (handshake.AdvertisableAddressResolver, *mockResolver, *mockResolver) { + tcpR := &mockResolver{ + resolveFunc: func(addr ma.Multiaddr) (ma.Multiaddr, error) { + return resolvedAddr, nil + }, + } + wssR := &mockResolver{} + return libp2p.NewCompositeAddressResolver(tcpR, wssR), tcpR, wssR + }, + wantTCPCalled: true, + wantResolved: true, + }, + { + name: "nil wss resolver", + addr: "/ip4/10.233.99.40/tcp/1635/tls/sni/*.libp2p.direct/ws/p2p/QmWbXocGMpfa8zApx9kCNwfmc35bbRJv136bdtuQjbR4wL", + setup: func() (handshake.AdvertisableAddressResolver, *mockResolver, *mockResolver) { + tcpR := &mockResolver{} + return libp2p.NewCompositeAddressResolver(tcpR, nil), tcpR, nil + }, + wantWSSCalled: false, + wantTCPCalled: false, + wantResolved: false, + }, + { + name: "nil tcp resolver", + addr: "/ip4/10.233.99.40/tcp/1634/p2p/QmWbXocGMpfa8zApx9kCNwfmc35bbRJv136bdtuQjbR4wL", + setup: func() (handshake.AdvertisableAddressResolver, *mockResolver, *mockResolver) { + wssR := &mockResolver{} + return libp2p.NewCompositeAddressResolver(nil, wssR), nil, wssR + }, + wantTCPCalled: false, + wantWSSCalled: false, + wantResolved: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + resolver, tcpResolver, wssResolver := tc.setup() + + addr, err := ma.NewMultiaddr(tc.addr) + if err != nil { + t.Fatal(err) + } + + got, err := resolver.Resolve(addr) + if !errors.Is(err, tc.wantErr) { + t.Errorf("got error %v, want %v", err, tc.wantErr) + } + + if tcpResolver != nil && tc.wantTCPCalled != tcpResolver.called { + t.Errorf("tcpResolver called: got %v, want %v", tcpResolver.called, tc.wantTCPCalled) + } + if wssResolver != nil && tc.wantWSSCalled != wssResolver.called { + t.Errorf("wssResolver called: got %v, want %v", wssResolver.called, tc.wantWSSCalled) + } + + if tc.wantResolved { + if !got.Equal(resolvedAddr) { + t.Errorf("got resolved addr %s, want %s", got, resolvedAddr) + } + } else { + if !got.Equal(addr) { + t.Errorf("got addr %s, want %s", got, addr) + } + } + }) + } +} diff --git a/pkg/p2p/libp2p/connections_test.go b/pkg/p2p/libp2p/connections_test.go index 42187edca9a..27ba4b7549d 100644 --- a/pkg/p2p/libp2p/connections_test.go +++ b/pkg/p2p/libp2p/connections_test.go @@ -37,6 +37,8 @@ import ( bhost "github.com/libp2p/go-libp2p/p2p/host/basic" swarmt "github.com/libp2p/go-libp2p/p2p/net/swarm/testing" ma "github.com/multiformats/go-multiaddr" + + libp2pmock "github.com/ethersphere/bee/v2/pkg/p2p/libp2p/mock" ) const ( @@ -86,6 +88,42 @@ func TestConnectDisconnect(t *testing.T) { expectPeersEventually(t, s1) } +// TestConnectSelf verifies that a service cannot connect to itself, +// preventing self-connection attempts. +func TestConnectSelf(t *testing.T) { + t.Parallel() + + ctx := t.Context() + + s1, _ := newService(t, 1, libp2pServiceOpts{libp2pOpts: libp2p.Options{ + FullNode: true, + }}) + + // Get own underlay addresses + addr := serviceUnderlayAddress(t, s1) + + // Attempt to connect to self + bzzAddr, err := s1.Connect(ctx, addr) + + // Should return an error + if err == nil { + t.Fatal("expected error when connecting to self, got nil") + } + + // Should contain "cannot connect to self" in error message + if !strings.Contains(err.Error(), "cannot connect to self") { + t.Fatalf("expected 'cannot connect to self' error, got: %v", err) + } + + // bzzAddr should be nil + if bzzAddr != nil { + t.Fatal("expected nil bzz address when connecting to self") + } + + // Verify no peers are connected + expectPeers(t, s1) +} + func TestConnectToLightPeer(t *testing.T) { t.Parallel() @@ -498,6 +536,15 @@ func TestConnectWithEnabledWSTransports(t *testing.T) { }, }) + defer func() { + if err := s1.Close(); err != nil { + t.Errorf("s1.Close: %v", err) + } + if err := s2.Close(); err != nil { + t.Errorf("s2.Close: %v", err) + } + }() + addr := serviceUnderlayAddress(t, s1) if _, err := s2.Connect(ctx, addr); err != nil { @@ -774,6 +821,85 @@ func TestTopologyNotifier(t *testing.T) { waitAddrSet(t, &n2disconnectedPeer.Address, &mtx, overlay1) } +func TestConnectWithAutoTLS(t *testing.T) { + t.Parallel() + + certLoaded := make(chan struct{}) + mockCertMgr := libp2pmock.NewMockP2PForgeCertMgr(func() { + close(certLoaded) + }) + + s1, _ := newService(t, 1, libp2pServiceOpts{ + libp2pOpts: libp2p.Options{ + EnableWS: true, + EnableWSS: true, + FullNode: true, + WSSAddr: ":0", + NATAddr: "127.0.0.1:1635", + NATWSSAddr: "127.0.0.1:1635", + }, + autoTLSCertManager: mockCertMgr, + }) + + select { + case <-certLoaded: + case <-time.After(time.Second): + t.Fatal("onCertLoaded callback was not triggered") + } + + if s1 == nil { + t.Fatal("service should not be nil") + } +} + +func TestConnectWithAutoTLSAndWSTransports(t *testing.T) { + t.Parallel() + + ctx := t.Context() + + s1, overlay1 := newService(t, 1, libp2pServiceOpts{ + libp2pOpts: libp2p.Options{ + EnableWS: true, + EnableWSS: true, + FullNode: true, + WSSAddr: ":0", + NATAddr: "127.0.0.1:1635", + NATWSSAddr: "127.0.0.1:1635", + }, + autoTLSCertManager: libp2pmock.NewMockP2PForgeCertMgr(nil), + }) + + s2, overlay2 := newService(t, 1, libp2pServiceOpts{ + libp2pOpts: libp2p.Options{ + EnableWS: true, + EnableWSS: true, + FullNode: true, + WSSAddr: ":0", + NATAddr: "127.0.0.1:1636", + NATWSSAddr: "127.0.0.1:1636", + }, + autoTLSCertManager: libp2pmock.NewMockP2PForgeCertMgr(nil), + }) + + defer func() { + if err := s1.Close(); err != nil { + t.Errorf("s1.Close: %v", err) + } + if err := s2.Close(); err != nil { + t.Errorf("s2.Close: %v", err) + } + }() + + addr := serviceUnderlayAddress(t, s1) + + if _, err := s2.Connect(ctx, addr); err != nil { + t.Fatal(err) + } + + expectPeers(t, s2, overlay1) + expectPeersEventually(t, s1, overlay2) +} + // TestTopologyAnnounce checks that announcement // works correctly for full nodes and light nodes. func TestTopologyAnnounce(t *testing.T) { @@ -895,8 +1021,6 @@ func TestTopologyAnnounce(t *testing.T) { } func TestTopologyOverSaturated(t *testing.T) { - t.Parallel() - var ( mtx sync.Mutex ctx = context.Background() @@ -1007,8 +1131,6 @@ func TestWithDisconnectStreams(t *testing.T) { } func TestWithBlocklistStreams(t *testing.T) { - t.Parallel() - ctx := t.Context() s1, overlay1 := newService(t, 1, libp2pServiceOpts{libp2pOpts: libp2p.Options{ diff --git a/pkg/p2p/libp2p/export_test.go b/pkg/p2p/libp2p/export_test.go index f78b9f21275..c85a85b54ae 100644 --- a/pkg/p2p/libp2p/export_test.go +++ b/pkg/p2p/libp2p/export_test.go @@ -42,3 +42,17 @@ func WithHostFactory(factory func(...libp2pm.Option) (host.Host, error)) Options hostFactory: factory, } } + +func WithAutoTLSCertManager(m autoTLSCertManager) Options { + return Options{ + autoTLSCertManager: m, + } +} + +func SetAutoTLSCertManager(o *Options, m autoTLSCertManager) { + o.autoTLSCertManager = m +} + +type AutoTLSCertManager = autoTLSCertManager + +var NewCompositeAddressResolver = newCompositeAddressResolver diff --git a/pkg/p2p/libp2p/libp2p.go b/pkg/p2p/libp2p/libp2p.go index 7a537913099..6bab9b8214c 100644 --- a/pkg/p2p/libp2p/libp2p.go +++ b/pkg/p2p/libp2p/libp2p.go @@ -7,10 +7,13 @@ package libp2p import ( "context" "crypto/ecdsa" + "crypto/tls" "errors" "fmt" "net" + "net/netip" "os" + "path/filepath" "runtime" "slices" "strconv" @@ -19,6 +22,7 @@ import ( "time" ocprom "contrib.go.opencensus.io/exporter/prometheus" + "github.com/caddyserver/certmagic" "github.com/coreos/go-semver/semver" "github.com/ethersphere/bee/v2" "github.com/ethersphere/bee/v2/pkg/addressbook" @@ -31,12 +35,14 @@ import ( "github.com/ethersphere/bee/v2/pkg/p2p/libp2p/internal/breaker" "github.com/ethersphere/bee/v2/pkg/p2p/libp2p/internal/handshake" "github.com/ethersphere/bee/v2/pkg/p2p/libp2p/internal/reacher" + libp2pmock "github.com/ethersphere/bee/v2/pkg/p2p/libp2p/mock" "github.com/ethersphere/bee/v2/pkg/storage" "github.com/ethersphere/bee/v2/pkg/swarm" "github.com/ethersphere/bee/v2/pkg/topology" "github.com/ethersphere/bee/v2/pkg/topology/lightnode" "github.com/ethersphere/bee/v2/pkg/tracing" "github.com/libp2p/go-libp2p" + "github.com/libp2p/go-libp2p/config" "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/event" "github.com/libp2p/go-libp2p/core/host" @@ -52,11 +58,16 @@ import ( libp2pping "github.com/libp2p/go-libp2p/p2p/protocol/ping" "github.com/libp2p/go-libp2p/p2p/transport/tcp" ws "github.com/libp2p/go-libp2p/p2p/transport/websocket" + libp2prate "github.com/libp2p/go-libp2p/x/rate" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr/net" "github.com/multiformats/go-multistream" "github.com/prometheus/client_golang/prometheus" "go.uber.org/atomic" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + + p2pforge "github.com/ipshipyard/p2p-forge/client" ) // loggerName is the tree path name of the logger for this package. @@ -84,33 +95,35 @@ const ( ) type Service struct { - ctx context.Context - host host.Host - natManager basichost.NATManager - natAddrResolver *staticAddressResolver - autonatDialer host.Host - pingDialer host.Host - libp2pPeerstore peerstore.Peerstore - metrics metrics - networkID uint64 - handshakeService *handshake.Service - addressbook addressbook.Putter - peers *peerRegistry - connectionBreaker breaker.Interface - blocklist *blocklist.Blocklist - protocols []p2p.ProtocolSpec - notifier p2p.PickyNotifier - logger log.Logger - tracer *tracing.Tracer - ready chan struct{} - halt chan struct{} - lightNodes lightnodes - lightNodeLimit int - protocolsmu sync.RWMutex - reacher p2p.Reacher - networkStatus atomic.Int32 - HeadersRWTimeout time.Duration - autoNAT autonat.AutoNAT + ctx context.Context + host host.Host + natManager basichost.NATManager + autonatDialer host.Host + pingDialer host.Host + libp2pPeerstore peerstore.Peerstore + metrics metrics + networkID uint64 + handshakeService *handshake.Service + addressbook addressbook.Putter + peers *peerRegistry + connectionBreaker breaker.Interface + blocklist *blocklist.Blocklist + protocols []p2p.ProtocolSpec + notifier p2p.PickyNotifier + logger log.Logger + tracer *tracing.Tracer + ready chan struct{} + halt chan struct{} + lightNodes lightnodes + lightNodeLimit int + protocolsmu sync.RWMutex + reacher p2p.Reacher + networkStatus atomic.Int32 + HeadersRWTimeout time.Duration + autoNAT autonat.AutoNAT + enableWS bool + autoTLSCertManager autoTLSCertManager + zapLogger *zap.Logger } type lightnodes interface { @@ -121,52 +134,72 @@ type lightnodes interface { EachPeer(pf topology.EachPeerFunc) error } +// autoTLSCertManager defines the interface for managing TLS certificates. +type autoTLSCertManager interface { + Start() error + Stop() + TLSConfig() *tls.Config + AddressFactory() config.AddrsFactory +} + type Options struct { - PrivateKey *ecdsa.PrivateKey - NATAddr string - EnableWS bool - FullNode bool - LightNodeLimit int - WelcomeMessage string - Nonce []byte - ValidateOverlay bool - hostFactory func(...libp2p.Option) (host.Host, error) - HeadersRWTimeout time.Duration - Registry *prometheus.Registry + PrivateKey *ecdsa.PrivateKey + NATAddr string + NATWSSAddr string + EnableWS bool + EnableWSS bool + WSSAddr string + AutoTLSStorageDir string + AutoTLSCAEndpoint string + AutoTLSDomain string + AutoTLSRegistrationEndpoint string + FullNode bool + LightNodeLimit int + WelcomeMessage string + Nonce []byte + ValidateOverlay bool + hostFactory func(...libp2p.Option) (host.Host, error) + HeadersRWTimeout time.Duration + Registry *prometheus.Registry + autoTLSCertManager autoTLSCertManager } -func New(ctx context.Context, signer beecrypto.Signer, networkID uint64, overlay swarm.Address, addr string, ab addressbook.Putter, storer storage.StateStorer, lightNodes *lightnode.Container, logger log.Logger, tracer *tracing.Tracer, o Options) (*Service, error) { - host, port, err := net.SplitHostPort(addr) +func New(ctx context.Context, signer beecrypto.Signer, networkID uint64, overlay swarm.Address, addr string, ab addressbook.Putter, storer storage.StateStorer, lightNodes *lightnode.Container, logger log.Logger, tracer *tracing.Tracer, o Options) (s *Service, returnErr error) { + logger = logger.WithName(loggerName).Register() + + parsedAddr, err := parseAddress(addr) if err != nil { - return nil, fmt.Errorf("address: %w", err) + return nil, err } - ip4Addr := "0.0.0.0" - ip6Addr := "::" + var listenAddrs []string - if host != "" { - ip := net.ParseIP(host) - if ip4 := ip.To4(); ip4 != nil { - ip4Addr = ip4.String() - ip6Addr = "" - } else if ip6 := ip.To16(); ip6 != nil { - ip6Addr = ip6.String() - ip4Addr = "" + if parsedAddr.IP4 != "" { + listenAddrs = append(listenAddrs, fmt.Sprintf("/ip4/%s/tcp/%s", parsedAddr.IP4, parsedAddr.Port)) + if o.EnableWS { + listenAddrs = append(listenAddrs, fmt.Sprintf("/ip4/%s/tcp/%s/ws", parsedAddr.IP4, parsedAddr.Port)) } } - var listenAddrs []string - if ip4Addr != "" { - listenAddrs = append(listenAddrs, fmt.Sprintf("/ip4/%s/tcp/%s", ip4Addr, port)) + if parsedAddr.IP6 != "" { + listenAddrs = append(listenAddrs, fmt.Sprintf("/ip6/%s/tcp/%s", parsedAddr.IP6, parsedAddr.Port)) if o.EnableWS { - listenAddrs = append(listenAddrs, fmt.Sprintf("/ip4/%s/tcp/%s/ws", ip4Addr, port)) + listenAddrs = append(listenAddrs, fmt.Sprintf("/ip6/%s/tcp/%s/ws", parsedAddr.IP6, parsedAddr.Port)) } } - if ip6Addr != "" { - listenAddrs = append(listenAddrs, fmt.Sprintf("/ip6/%s/tcp/%s", ip6Addr, port)) - if o.EnableWS { - listenAddrs = append(listenAddrs, fmt.Sprintf("/ip6/%s/tcp/%s/ws", ip6Addr, port)) + if o.EnableWSS { + parsedWssAddr, err := parseAddress(o.WSSAddr) + if err != nil { + return nil, err + } + + if parsedWssAddr.IP4 != "" { + listenAddrs = append(listenAddrs, fmt.Sprintf("/ip4/%s/tcp/%s/tls/sni/*.%s/ws", parsedWssAddr.IP4, parsedWssAddr.Port, o.AutoTLSDomain)) + } + + if parsedWssAddr.IP6 != "" { + listenAddrs = append(listenAddrs, fmt.Sprintf("/ip6/%s/tcp/%s/tls/sni/*.%s/ws", parsedWssAddr.IP6, parsedWssAddr.Port, o.AutoTLSDomain)) } } @@ -208,13 +241,141 @@ func New(ctx context.Context, signer beecrypto.Signer, networkID uint64, overlay return nil, err } - rm, err := rcmgr.NewResourceManager(limiter, rcmgr.WithTraceReporter(str)) + limitPerIp := rcmgr.WithLimitPerSubnet( + []rcmgr.ConnLimitPerSubnet{{PrefixLength: 32, ConnCount: 200}}, // IPv4 /32 (Single IP) -> 200 conns + []rcmgr.ConnLimitPerSubnet{{PrefixLength: 56, ConnCount: 200}}, // IPv6 /56 subnet -> 200 conns + ) + + // Custom rate limiter for connection attempts + // 20 peers cluster adaptation: + // Allow bursts of connection attempts (e.g. restart) but prevent DDOS. + connLimiter := &libp2prate.Limiter{ + // Allow unlimited local connections (same as default) + NetworkPrefixLimits: []libp2prate.PrefixLimit{ + {Prefix: netip.MustParsePrefix("127.0.0.0/8"), Limit: libp2prate.Limit{}}, + {Prefix: netip.MustParsePrefix("::1/128"), Limit: libp2prate.Limit{}}, + }, + GlobalLimit: libp2prate.Limit{}, // Unlimited global + SubnetRateLimiter: libp2prate.SubnetLimiter{ + IPv4SubnetLimits: []libp2prate.SubnetLimit{ + { + PrefixLength: 32, // Apply limits per individual IPv4 address (/32) + // Allow 10 connection attempts per second per IP, burst up to 40 + Limit: libp2prate.Limit{RPS: 10.0, Burst: 40}, + }, + }, + IPv6SubnetLimits: []libp2prate.SubnetLimit{ + { + PrefixLength: 56, // Apply limits per /56 IPv6 subnet + // Allow 10 connection attempts per second per IP, burst up to 40 + // Subnet-level limiting prevents flooding from multiple addresses in the same block. + Limit: libp2prate.Limit{RPS: 10.0, Burst: 40}, + }, + }, + // Duration to retain state for an IP or subnet after it becomes inactive. + GracePeriod: 10 * time.Second, + }, + } + + rm, err := rcmgr.NewResourceManager(limiter, rcmgr.WithTraceReporter(str), limitPerIp, rcmgr.WithConnRateLimiters(connLimiter)) if err != nil { return nil, err } var natManager basichost.NATManager + var certManager autoTLSCertManager + var zapLogger *zap.Logger + + // AutoTLS is only needed for WSS + enableAutoTLS := o.EnableWSS + + if enableAutoTLS { + if o.autoTLSCertManager != nil { + certManager = o.autoTLSCertManager + } else { + // create a zap logger needed for cert manager to be as close to + // swarm logger as possible + l, err := zap.Config{ + Level: zap.NewAtomicLevelAt(zap.DebugLevel), + Development: false, + Sampling: &zap.SamplingConfig{ + Initial: 100, + Thereafter: 100, + }, + Encoding: "json", + EncoderConfig: zapcore.EncoderConfig{ + TimeKey: "time", + LevelKey: "level", + NameKey: "logger", + CallerKey: "caller", + FunctionKey: zapcore.OmitKey, + MessageKey: "msg", + StacktraceKey: "stacktrace", + LineEnding: zapcore.DefaultLineEnding, + EncodeLevel: zapcore.LowercaseLevelEncoder, + EncodeTime: zapcore.EpochTimeEncoder, + EncodeDuration: zapcore.SecondsDurationEncoder, + EncodeCaller: zapcore.ShortCallerEncoder, + }, + OutputPaths: []string{"stderr"}, + ErrorOutputPaths: []string{"stderr"}, + }.Build() + if err != nil { + return nil, err + } + + // assing zap logger as it needs to be synced when the service stops + zapLogger = l + + defer func() { + _ = zapLogger.Sync() + }() + + // Use AutoTLS storage dir with domain subdir for easier management + // of different registers. + storagePath := filepath.Join(o.AutoTLSStorageDir, o.AutoTLSDomain) + + if err := os.MkdirAll(storagePath, 0700); err != nil { + return nil, fmt.Errorf("create certificate storage directory %s: %w", storagePath, err) + } + + certManager, err = p2pforge.NewP2PForgeCertMgr( + p2pforge.WithForgeDomain(o.AutoTLSDomain), + p2pforge.WithForgeRegistrationEndpoint(o.AutoTLSRegistrationEndpoint), + p2pforge.WithCAEndpoint(o.AutoTLSCAEndpoint), + p2pforge.WithCertificateStorage(&certmagic.FileStorage{Path: storagePath}), + p2pforge.WithLogger(zapLogger.Sugar()), + p2pforge.WithUserAgent(userAgent()), + p2pforge.WithAllowPrivateForgeAddrs(), + p2pforge.WithRegistrationDelay(0), + p2pforge.WithOnCertLoaded(func() { + logger.Info("auto tls certificate is loaded") + }), + p2pforge.WithOnCertRenewed(func() { + logger.Info("auto tls certificate is renewed") + }), + ) + if err != nil { + return nil, fmt.Errorf("initialize AutoTLS: %w", err) + } + } + + defer func() { + if returnErr != nil { + // certificate manager has to be stopped if service is not + // constructed + certManager.Stop() + } + }() + + if err := certManager.Start(); err != nil { + return nil, fmt.Errorf("start AutoTLS certificate manager: %w", err) + } + + logger.Info("AutoTLS certificate manager initialized") + } + opts := []libp2p.Option{ libp2p.ListenAddrStrings(listenAddrs...), security, @@ -224,7 +385,7 @@ func New(ctx context.Context, signer beecrypto.Signer, networkID uint64, overlay libp2p.ResourceManager(rm), } - if o.NATAddr == "" { + if o.NATAddr == "" && o.NATWSSAddr == "" { opts = append(opts, libp2p.NATManager(func(n network.Network) basichost.NATManager { natManager = basichost.NewNATManager(n) @@ -247,10 +408,49 @@ func New(ctx context.Context, signer beecrypto.Signer, networkID uint64, overlay libp2p.Transport(tcp.NewTCPTransport, tcp.DisableReuseport()), } - if o.EnableWS { + var tcpResolver handshake.AdvertisableAddressResolver + if o.NATAddr != "" { + r, err := newStaticAddressResolver(o.NATAddr, net.LookupIP) + if err != nil { + return nil, fmt.Errorf("static nat: %w", err) + } + tcpResolver = r + } + + var wssResolver handshake.AdvertisableAddressResolver + if o.EnableWSS && o.NATWSSAddr != "" { + r, err := newStaticAddressResolver(o.NATWSSAddr, net.LookupIP) + if err != nil { + return nil, fmt.Errorf("static wss nat: %w", err) + } + wssResolver = r + } + + if o.EnableWSS { + wsOpt := ws.WithTLSConfig(certManager.TLSConfig()) + transports = append(transports, libp2p.Transport(ws.New, wsOpt)) + } else if o.EnableWS { transports = append(transports, libp2p.Transport(ws.New)) } + compositeResolver := newCompositeAddressResolver(tcpResolver, wssResolver) + + var addrFactory config.AddrsFactory + if o.EnableWSS { + certManagerFactory := certManager.AddressFactory() + addrFactory = func(addrs []ma.Multiaddr) []ma.Multiaddr { + addrs = includeNatResolvedAddresses(addrs, compositeResolver, logger) + addrs = certManagerFactory(addrs) + return addrs + } + } else { + addrFactory = func(addrs []ma.Multiaddr) []ma.Multiaddr { + return includeNatResolvedAddresses(addrs, compositeResolver, logger) + } + } + + opts = append(opts, libp2p.AddrsFactory(addrFactory)) + opts = append(opts, transports...) if o.hostFactory == nil { @@ -263,6 +463,18 @@ func New(ctx context.Context, signer beecrypto.Signer, networkID uint64, overlay return nil, err } + if enableAutoTLS { + switch cm := certManager.(type) { + case *p2pforge.P2PForgeCertMgr: + cm.ProvideHost(h) + case *libp2pmock.MockP2PForgeCertMgr: + if err := cm.ProvideHost(h); err != nil { + return nil, fmt.Errorf("failed to provide host to MockP2PForgeCertMgr: %w", err) + } + default: + return nil, fmt.Errorf("unknown cert manager type") + } + } // Support same non default security and transport options as // original host. dialer, err := o.hostFactory(append(transports, security)...) @@ -270,7 +482,13 @@ func New(ctx context.Context, signer beecrypto.Signer, networkID uint64, overlay return nil, err } - options := []autonat.Option{autonat.EnableService(dialer.Network())} + if o.HeadersRWTimeout == 0 { + o.HeadersRWTimeout = defaultHeadersRWTimeout + } + + options := []autonat.Option{ + autonat.EnableService(dialer.Network()), + } val, err := strconv.ParseBool(reachabilityOverridePublic) if err != nil { @@ -288,25 +506,7 @@ func New(ctx context.Context, signer beecrypto.Signer, networkID uint64, overlay return nil, fmt.Errorf("autonat: %w", err) } - if o.HeadersRWTimeout == 0 { - o.HeadersRWTimeout = defaultHeadersRWTimeout - } - - var advertisableAddresser handshake.AdvertisableAddressResolver - var natAddrResolver *staticAddressResolver - if o.NATAddr == "" { - advertisableAddresser = &UpnpAddressResolver{ - host: h, - } - } else { - natAddrResolver, err = newStaticAddressResolver(o.NATAddr, net.LookupIP) - if err != nil { - return nil, fmt.Errorf("static nat: %w", err) - } - advertisableAddresser = natAddrResolver - } - - handshakeService, err := handshake.New(signer, advertisableAddresser, overlay, networkID, o.FullNode, o.Nonce, newHostAddresser(h), o.WelcomeMessage, o.ValidateOverlay, h.ID(), logger) + handshakeService, err := handshake.New(signer, newCompositeAddressResolver(tcpResolver, wssResolver), overlay, networkID, o.FullNode, o.Nonce, newHostAddresser(h), o.WelcomeMessage, o.ValidateOverlay, h.ID(), logger) if err != nil { return nil, fmt.Errorf("handshake service: %w", err) } @@ -322,28 +522,30 @@ func New(ctx context.Context, signer beecrypto.Signer, networkID uint64, overlay } peerRegistry := newPeerRegistry() - s := &Service{ - ctx: ctx, - host: h, - natManager: natManager, - natAddrResolver: natAddrResolver, - autonatDialer: dialer, - pingDialer: pingDialer, - handshakeService: handshakeService, - libp2pPeerstore: libp2pPeerstore, - metrics: newMetrics(), - networkID: networkID, - peers: peerRegistry, - addressbook: ab, - blocklist: blocklist.NewBlocklist(storer), - logger: logger.WithName(loggerName).Register(), - tracer: tracer, - connectionBreaker: breaker.NewBreaker(breaker.Options{}), // use default options - ready: make(chan struct{}), - halt: make(chan struct{}), - lightNodes: lightNodes, - HeadersRWTimeout: o.HeadersRWTimeout, - autoNAT: autoNAT, + s = &Service{ + ctx: ctx, + host: h, + natManager: natManager, + autonatDialer: dialer, + pingDialer: pingDialer, + handshakeService: handshakeService, + libp2pPeerstore: libp2pPeerstore, + metrics: newMetrics(), + networkID: networkID, + peers: peerRegistry, + addressbook: ab, + blocklist: blocklist.NewBlocklist(storer), + logger: logger, + tracer: tracer, + connectionBreaker: breaker.NewBreaker(breaker.Options{}), // use default options + ready: make(chan struct{}), + halt: make(chan struct{}), + lightNodes: lightNodes, + HeadersRWTimeout: o.HeadersRWTimeout, + autoNAT: autoNAT, + enableWS: o.EnableWS, + autoTLSCertManager: certManager, + zapLogger: zapLogger, } peerRegistry.setDisconnecter(s) @@ -369,6 +571,37 @@ func New(ctx context.Context, signer beecrypto.Signer, networkID uint64, overlay return s, nil } +type parsedAddress struct { + IP4 string + IP6 string + Port string +} + +func parseAddress(addr string) (*parsedAddress, error) { + host, port, err := net.SplitHostPort(addr) + if err != nil { + return nil, fmt.Errorf("parse address %s: %w", addr, err) + } + + res := &parsedAddress{ + IP4: "0.0.0.0", + IP6: "::", + Port: port, + } + + if host != "" { + ip := net.ParseIP(host) + if ip4parsed := ip.To4(); ip4parsed != nil { + res.IP4 = ip4parsed.String() + res.IP6 = "" + } else if ip6parsed := ip.To16(); ip6parsed != nil { + res.IP6 = ip6parsed.String() + res.IP4 = "" + } + } + return res, nil +} + func (s *Service) reachabilityWorker() error { sub, err := s.host.EventBus().Subscribe([]any{new(event.EvtLocalReachabilityChanged)}) if err != nil { @@ -413,7 +646,7 @@ func (s *Service) handleIncoming(stream network.Stream) { peerID := stream.Conn().RemotePeer() handshakeStream := newStream(stream, s.metrics) - peerMultiaddrs, err := s.peerMultiaddrs(s.ctx, stream.Conn(), peerID) + peerMultiaddrs, err := s.peerMultiaddrs(s.ctx, peerID) if err != nil { s.logger.Debug("stream handler: handshake: build remote multiaddrs", "peer_id", peerID, "error", err) s.logger.Error(nil, "stream handler: handshake: build remote multiaddrs", "peer_id", peerID) @@ -443,7 +676,7 @@ func (s *Service) handleIncoming(stream network.Stream) { s.logger.Debug("stream handler: blocklisting: exists failed", "peer_address", overlay, "error", err) s.logger.Error(nil, "stream handler: internal error while connecting with peer", "peer_address", overlay) _ = handshakeStream.Reset() - _ = s.host.Network().ClosePeer(peerID) + _ = stream.Conn().Close() return } @@ -459,7 +692,7 @@ func (s *Service) handleIncoming(stream network.Stream) { if err = handshakeStream.FullClose(); err != nil { s.logger.Debug("stream handler: could not close stream", "peer_address", overlay, "error", err) s.logger.Error(nil, "stream handler: unable to handshake with peer", "peer_address", overlay) - _ = s.Disconnect(overlay, "unable to close handshake stream") + _ = stream.Conn().Close() } return } @@ -554,7 +787,7 @@ func (s *Service) handleIncoming(stream network.Stream) { return } - s.notifyReacherConnected(stream, overlay, peerID) + s.notifyReacherConnected(overlay, peerID) peerUserAgent := appendSpace(s.peerUserAgent(s.ctx, peerID)) s.networkStatus.Store(int32(p2p.NetworkStatusAvailable)) @@ -563,16 +796,15 @@ func (s *Service) handleIncoming(stream network.Stream) { s.logger.Debug("stream handler: successfully connected to peer (inbound)", "address", i.BzzAddress.Overlay, "light", i.LightString(), "user_agent", peerUserAgent) } -func (s *Service) notifyReacherConnected(stream network.Stream, overlay swarm.Address, peerID libp2ppeer.ID) { +func (s *Service) notifyReacherConnected(overlay swarm.Address, peerID libp2ppeer.ID) { if s.reacher == nil { return } peerAddrs := s.host.Peerstore().Addrs(peerID) - connectionAddr := stream.Conn().RemoteMultiaddr() - bestAddr := bzz.SelectBestAdvertisedAddress(peerAddrs, connectionAddr) + bestAddr := bzz.SelectBestAdvertisedAddress(peerAddrs, nil) - s.logger.Debug("selected reacher address", "peer_id", peerID, "selected_addr", bestAddr.String(), "connection_addr", connectionAddr.String(), "advertised_count", len(peerAddrs)) + s.logger.Debug("selected reacher address", "peer_id", peerID, "selected_addr", bestAddr.String(), "advertised_count", len(peerAddrs)) underlay, err := buildFullMA(bestAddr, peerID) if err != nil { @@ -680,23 +912,37 @@ func (s *Service) AddProtocol(p p2p.ProtocolSpec) (err error) { } func (s *Service) Addresses() (addresses []ma.Multiaddr, err error) { - for _, addr := range s.host.Addrs() { - a, err := buildUnderlayAddress(addr, s.host.ID()) - if err != nil { - return nil, err - } + return buildFullMAs(s.host.Addrs(), s.host.ID()) +} - addresses = append(addresses, a) +func includeNatResolvedAddresses(addrs []ma.Multiaddr, advertisableAddresser handshake.AdvertisableAddressResolver, logger log.Logger) (addresses []ma.Multiaddr) { + if advertisableAddresser == nil { + return addrs } - if s.natAddrResolver != nil && len(addresses) > 0 { - a, err := s.natAddrResolver.Resolve(addresses[0]) + + allAddrs := make([]ma.Multiaddr, 0, len(addrs)) + + for _, addr := range addrs { + allAddrs = append(allAddrs, addr) + + resolved, err := advertisableAddresser.Resolve(addr) if err != nil { - return nil, err + logger.Warning("could not resolve address", "addr", addr, "error", err) + continue + } + + if resolved.Equal(addr) { + continue + } + + if !slices.ContainsFunc(allAddrs, func(a ma.Multiaddr) bool { + return resolved.Equal(a) + }) { + allAddrs = append(allAddrs, resolved) } - addresses = append(addresses, a) } - return addresses, nil + return allAddrs } func (s *Service) NATManager() basichost.NATManager { @@ -733,16 +979,6 @@ func buildHostAddress(peerID libp2ppeer.ID) (ma.Multiaddr, error) { return ma.NewMultiaddr(fmt.Sprintf("/p2p/%s", peerID.String())) } -func buildUnderlayAddress(addr ma.Multiaddr, peerID libp2ppeer.ID) (ma.Multiaddr, error) { - // Build host multiaddress - hostAddr, err := buildHostAddress(peerID) - if err != nil { - return nil, err - } - - return addr.Encapsulate(hostAddr), nil -} - func (s *Service) Connect(ctx context.Context, addrs []ma.Multiaddr) (address *bzz.Address, err error) { loggerV1 := s.logger.V(1).Register() @@ -753,6 +989,7 @@ func (s *Service) Connect(ctx context.Context, addrs []ma.Multiaddr) (address *b var info *libp2ppeer.AddrInfo var peerID libp2ppeer.ID var connectErr error + skippedSelf := false // Try to connect to each underlay address one by one. // @@ -770,6 +1007,13 @@ func (s *Service) Connect(ctx context.Context, addrs []ma.Multiaddr) (address *b info = ai peerID = ai.ID + // Check if attempting to connect to self + if peerID == s.host.ID() { + s.logger.Debug("skipping connection to self", "peer_id", peerID, "underlay", info.Addrs) + skippedSelf = true + continue + } + hostAddr, err := buildHostAddress(info.ID) if err != nil { return nil, fmt.Errorf("build host address: %w", err) @@ -799,7 +1043,12 @@ func (s *Service) Connect(ctx context.Context, addrs []ma.Multiaddr) (address *b } if connectErr != nil { - return nil, fmt.Errorf("libp2p connect: %w", err) + return nil, fmt.Errorf("libp2p connect: %w", connectErr) + } + + // If we skipped all addresses due to self-connection, return an error + if skippedSelf { + return nil, fmt.Errorf("cannot connect to self") } if info == nil { @@ -814,7 +1063,7 @@ func (s *Service) Connect(ctx context.Context, addrs []ma.Multiaddr) (address *b handshakeStream := newStream(stream, s.metrics) - peerMultiaddrs, err := s.peerMultiaddrs(ctx, stream.Conn(), peerID) + peerMultiaddrs, err := s.peerMultiaddrs(ctx, peerID) if err != nil { _ = handshakeStream.Reset() _ = s.host.Network().ClosePeer(peerID) @@ -871,6 +1120,10 @@ func (s *Service) Connect(ctx context.Context, addrs []ma.Multiaddr) (address *b return nil, fmt.Errorf("connect full close %w", err) } + if !s.peers.Exists(overlay) { + return nil, p2p.ErrPeerNotFound + } + if i.FullNode { err = s.addressbook.Put(overlay, *i.BzzAddress) if err != nil { @@ -899,7 +1152,7 @@ func (s *Service) Connect(ctx context.Context, addrs []ma.Multiaddr) (address *b s.metrics.CreatedConnectionCount.Inc() - s.notifyReacherConnected(stream, overlay, peerID) + s.notifyReacherConnected(overlay, peerID) peerUA := appendSpace(s.peerUserAgent(ctx, peerID)) loggerV1.Debug("successfully connected to peer (outbound)", "addresses", i.BzzAddress.ShortString(), "light", i.LightString(), "user_agent", peerUA) @@ -1004,6 +1257,12 @@ func (s *Service) NewStream(ctx context.Context, overlay swarm.Address, headers return nil, p2p.ErrPeerNotFound } + // Verify if we really have an active connection + if s.host.Network().Connectedness(peerID) != network.Connected { + _ = s.Disconnect(overlay, "registry-host mismatch in NewStream") + return nil, p2p.ErrPeerNotFound + } + streamlibp2p, err := s.newStreamForPeerID(ctx, peerID, protocolName, protocolVersion, streamName) if err != nil { return nil, fmt.Errorf("new stream for peerid: %w", err) @@ -1033,6 +1292,11 @@ func (s *Service) NewStream(ctx context.Context, overlay swarm.Address, headers func (s *Service) newStreamForPeerID(ctx context.Context, peerID libp2ppeer.ID, protocolName, protocolVersion, streamName string) (network.Stream, error) { swarmStreamName := p2p.NewSwarmStreamName(protocolName, protocolVersion, streamName) + + if s.host.Network().Connectedness(peerID) != network.Connected { + s.logger.Debug("newStreamForPeerID: host not connected to peer, this will trigger a dial", "peer_id", peerID, "protocol", swarmStreamName) + } + st, err := s.host.NewStream(ctx, peerID, protocol.ID(swarmStreamName)) if err != nil { if st != nil { @@ -1053,6 +1317,9 @@ func (s *Service) newStreamForPeerID(ctx context.Context, peerID libp2ppeer.ID, } func (s *Service) Close() error { + if s.autoTLSCertManager != nil { + s.autoTLSCertManager.Stop() + } if err := s.libp2pPeerstore.Close(); err != nil { return err } @@ -1077,6 +1344,9 @@ func (s *Service) Close() error { return err } } + if s.zapLogger != nil { + _ = s.zapLogger.Sync() + } return s.host.Close() } @@ -1194,24 +1464,11 @@ func (s *Service) determineCurrentNetworkStatus(err error) error { // peerMultiaddrs builds full multiaddresses for a peer given information from // libp2p host peerstore and falling back to the remote address from the // connection. -func (s *Service) peerMultiaddrs(ctx context.Context, conn network.Conn, peerID libp2ppeer.ID) ([]ma.Multiaddr, error) { +func (s *Service) peerMultiaddrs(ctx context.Context, peerID libp2ppeer.ID) ([]ma.Multiaddr, error) { waitPeersCtx, cancel := context.WithTimeout(ctx, peerstoreWaitAddrsTimeout) defer cancel() - peerMultiaddrs, err := buildFullMAs(waitPeerAddrs(waitPeersCtx, s.host.Peerstore(), peerID), peerID) - if err != nil { - return nil, fmt.Errorf("build peer multiaddrs: %w", err) - } - - if len(peerMultiaddrs) == 0 { - fullRemoteAddress, err := buildFullMA(conn.RemoteMultiaddr(), peerID) - if err != nil { - return nil, fmt.Errorf("build full remote peer multi address: %w", err) - } - peerMultiaddrs = append(peerMultiaddrs, fullRemoteAddress) - } - - return peerMultiaddrs, nil + return buildFullMAs(waitPeerAddrs(waitPeersCtx, s.host.Peerstore(), peerID), peerID) } var version270 = *semver.Must(semver.NewVersion("2.7.0")) @@ -1290,6 +1547,40 @@ func isNetworkOrHostUnreachableError(err error) bool { return false } +type compositeAddressResolver struct { + tcpResolver handshake.AdvertisableAddressResolver + wssResolver handshake.AdvertisableAddressResolver +} + +func newCompositeAddressResolver(tcpResolver, wssResolver handshake.AdvertisableAddressResolver) handshake.AdvertisableAddressResolver { + return &compositeAddressResolver{ + tcpResolver: tcpResolver, + wssResolver: wssResolver, + } +} + +func (c *compositeAddressResolver) Resolve(observedAddress ma.Multiaddr) (ma.Multiaddr, error) { + protocols := observedAddress.Protocols() + + containsProtocol := func(protocols []ma.Protocol, code int) bool { + return slices.ContainsFunc(protocols, func(p ma.Protocol) bool { return p.Code == code }) + } + + // ma.P_WSS protocol is deprecated, multiaddrs should comtain WS and TLS protocols for WSS + isWSS := containsProtocol(protocols, ma.P_WS) && containsProtocol(protocols, ma.P_TLS) + + if isWSS { + if c.wssResolver != nil { + return c.wssResolver.Resolve(observedAddress) + } + } else { + if c.tcpResolver != nil { + return c.tcpResolver.Resolve(observedAddress) + } + } + return observedAddress, nil +} + type hostAddresser struct { host host.Host } diff --git a/pkg/p2p/libp2p/libp2p_test.go b/pkg/p2p/libp2p/libp2p_test.go index 85208272f1c..ede9cbbcb79 100644 --- a/pkg/p2p/libp2p/libp2p_test.go +++ b/pkg/p2p/libp2p/libp2p_test.go @@ -27,13 +27,14 @@ import ( ) type libp2pServiceOpts struct { - Logger log.Logger - Addressbook addressbook.Interface - PrivateKey *ecdsa.PrivateKey - MockPeerKey *ecdsa.PrivateKey - libp2pOpts libp2p.Options - lightNodes *lightnode.Container - notifier p2p.PickyNotifier + Logger log.Logger + Addressbook addressbook.Interface + PrivateKey *ecdsa.PrivateKey + MockPeerKey *ecdsa.PrivateKey + libp2pOpts libp2p.Options + lightNodes *lightnode.Container + notifier p2p.PickyNotifier + autoTLSCertManager libp2p.AutoTLSCertManager } // newService constructs a new libp2p service. @@ -81,6 +82,10 @@ func newService(t *testing.T, networkID uint64, o libp2pServiceOpts) (s *libp2p. opts := o.libp2pOpts opts.Nonce = nonce + if o.autoTLSCertManager != nil { + libp2p.SetAutoTLSCertManager(&opts, o.autoTLSCertManager) + } + s, err = libp2p.New(ctx, crypto.NewDefaultSigner(swarmKey), networkID, overlay, addr, o.Addressbook, statestore, o.lightNodes, o.Logger, nil, opts) if err != nil { t.Fatal(err) @@ -128,7 +133,7 @@ func expectPeersEventually(t *testing.T, s *libp2p.Service, addrs ...swarm.Addre t.Helper() var peers []p2p.Peer - err := spinlock.Wait(time.Second, func() bool { + err := spinlock.Wait(5*time.Second, func() bool { peers = s.Peers() return len(peers) == len(addrs) diff --git a/pkg/p2p/libp2p/libp2ptest/buffer.go b/pkg/p2p/libp2p/libp2ptest/buffer.go new file mode 100644 index 00000000000..ed98e09b285 --- /dev/null +++ b/pkg/p2p/libp2p/libp2ptest/buffer.go @@ -0,0 +1,28 @@ +// Copyright 2025 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package libp2ptest + +import ( + "bytes" + "sync" +) + +// SafeBuffer is a thread-safe bytes.Buffer. +type SafeBuffer struct { + b bytes.Buffer + m sync.Mutex +} + +func (s *SafeBuffer) Write(p []byte) (n int, err error) { + s.m.Lock() + defer s.m.Unlock() + return s.b.Write(p) +} + +func (s *SafeBuffer) String() string { + s.m.Lock() + defer s.m.Unlock() + return s.b.String() +} diff --git a/pkg/p2p/libp2p/libp2ptest/service.go b/pkg/p2p/libp2p/libp2ptest/service.go new file mode 100644 index 00000000000..8a62531f342 --- /dev/null +++ b/pkg/p2p/libp2p/libp2ptest/service.go @@ -0,0 +1,69 @@ +// Copyright 2025 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package libp2ptest + +import ( + "context" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethersphere/bee/v2/pkg/addressbook" + "github.com/ethersphere/bee/v2/pkg/crypto" + "github.com/ethersphere/bee/v2/pkg/log" + "github.com/ethersphere/bee/v2/pkg/p2p/libp2p" + "github.com/ethersphere/bee/v2/pkg/statestore/mock" + "github.com/ethersphere/bee/v2/pkg/swarm" + "github.com/ethersphere/bee/v2/pkg/topology/lightnode" + "github.com/ethersphere/bee/v2/pkg/util/testutil" +) + +// NewLibp2pService creates a new libp2p service for testing purposes. +func NewLibp2pService(t *testing.T, networkID uint64, logger log.Logger) (*libp2p.Service, swarm.Address) { + t.Helper() + + swarmKey, err := crypto.GenerateSecp256k1Key() + if err != nil { + t.Fatal(err) + } + + nonce := common.HexToHash("0x1").Bytes() + + overlay, err := crypto.NewOverlayAddress(swarmKey.PublicKey, networkID, nonce) + if err != nil { + t.Fatal(err) + } + + addr := ":0" + + statestore := mock.NewStateStore() + ab := addressbook.New(statestore) + + libp2pKey, err := crypto.GenerateSecp256r1Key() + if err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + lightNodes := lightnode.NewContainer(overlay) + + opts := libp2p.Options{ + PrivateKey: libp2pKey, + Nonce: nonce, + FullNode: true, + NATAddr: "127.0.0.1:0", // Disable default NAT manager + } + + s, err := libp2p.New(ctx, crypto.NewDefaultSigner(swarmKey), networkID, overlay, addr, ab, statestore, lightNodes, logger, nil, opts) + if err != nil { + t.Fatal(err) + } + testutil.CleanupCloser(t, s) + + _ = s.Ready() + + return s, overlay +} diff --git a/pkg/p2p/libp2p/mock/mock_certmagic.go b/pkg/p2p/libp2p/mock/mock_certmagic.go new file mode 100644 index 00000000000..8337f26e8ab --- /dev/null +++ b/pkg/p2p/libp2p/mock/mock_certmagic.go @@ -0,0 +1,158 @@ +// Copyright 2025 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +package mock + +import ( + "context" + "crypto/tls" + "sync" + + _ "embed" + + "github.com/libp2p/go-libp2p/config" + "github.com/libp2p/go-libp2p/core/host" + ma "github.com/multiformats/go-multiaddr" +) + +//go:embed testdata/cert.pem +var certPEM []byte + +//go:embed testdata/key.pem +var keyPEM []byte + +// MockFileStorage is a minimal implementation of certmagic.FileStorage for testing. +type MockFileStorage struct { + path string +} + +func NewMockFileStorage(path string) *MockFileStorage { + return &MockFileStorage{path: path} +} + +func (m *MockFileStorage) Store(_ context.Context, _ string, _ []byte) error { + return nil +} + +func (m *MockFileStorage) Load(_ context.Context, _ string) ([]byte, error) { + return nil, nil +} + +func (m *MockFileStorage) Delete(_ context.Context, _ string) error { + return nil +} + +func (m *MockFileStorage) Exists(_ context.Context, _ string) (bool, error) { + return false, nil +} + +func (m *MockFileStorage) List(_ context.Context, _ string, _ bool) ([]string, error) { + return nil, nil +} + +func (m *MockFileStorage) Lock(_ context.Context, _ string) error { + return nil +} + +func (m *MockFileStorage) Unlock(_ context.Context, _ string) error { + return nil +} + +// MockCache is a minimal implementation of certmagic.Cache for testing. +type MockCache struct { + stopped bool + mu sync.Mutex +} + +func NewMockCache() *MockCache { + return &MockCache{} +} + +func (m *MockCache) Stop() { + m.mu.Lock() + defer m.mu.Unlock() + m.stopped = true +} + +// MockConfig is a minimal implementation of certmagic.Config for testing. +type MockConfig struct { + cache *MockCache +} + +func NewMockConfig() *MockConfig { + return &MockConfig{cache: NewMockCache()} +} + +func (m *MockConfig) TLSConfig() *tls.Config { + return &tls.Config{ + MinVersion: tls.VersionTLS12, + } +} + +// MockP2PForgeCertMgr is a mock implementation of p2pforge.P2PForgeCertMgr. +type MockP2PForgeCertMgr struct { + cache *MockCache + onCertLoaded func() + started bool + mu sync.Mutex + ProvideHost func(host.Host) error +} + +func NewMockP2PForgeCertMgr(onCertLoaded func()) *MockP2PForgeCertMgr { + return &MockP2PForgeCertMgr{ + cache: NewMockCache(), + onCertLoaded: onCertLoaded, + ProvideHost: func(_ host.Host) error { return nil }, + } +} + +func (m *MockP2PForgeCertMgr) Start() error { + m.mu.Lock() + defer m.mu.Unlock() + if m.started { + return nil + } + m.started = true + if m.onCertLoaded != nil { + go func() { + m.onCertLoaded() + }() + } + return nil +} +func (m *MockP2PForgeCertMgr) Stop() { + m.mu.Lock() + defer m.mu.Unlock() + if !m.started { + return + } + m.started = false + m.cache.Stop() +} + +func (m *MockP2PForgeCertMgr) TLSConfig() *tls.Config { + // Use tls.X509KeyPair to create a certificate from the hardcoded strings + cert, err := tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + // This should not fail if the strings are pasted correctly + return nil + } + + return &tls.Config{Certificates: []tls.Certificate{cert}} +} + +func (m *MockP2PForgeCertMgr) SetOnCertLoaded(cb func()) { + m.mu.Lock() + defer m.mu.Unlock() + m.onCertLoaded = cb +} + +func (m *MockP2PForgeCertMgr) AddressFactory() config.AddrsFactory { + return func(addrs []ma.Multiaddr) []ma.Multiaddr { + return addrs + } +} + +func (m *MockP2PForgeCertMgr) GetCache() *MockCache { + return m.cache +} diff --git a/pkg/p2p/libp2p/mock/mock_certmagic_test.go b/pkg/p2p/libp2p/mock/mock_certmagic_test.go new file mode 100644 index 00000000000..93dd0d159b6 --- /dev/null +++ b/pkg/p2p/libp2p/mock/mock_certmagic_test.go @@ -0,0 +1,112 @@ +// Copyright 2025 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package mock_test + +import ( + "crypto/tls" + "testing" + "time" + + "github.com/ethersphere/bee/v2/pkg/p2p/libp2p/mock" + ma "github.com/multiformats/go-multiaddr" +) + +func TestMockP2PForgeCertMgr_Start(t *testing.T) { + certLoaded := make(chan struct{}) + mgr := mock.NewMockP2PForgeCertMgr(func() { + close(certLoaded) + }) + + if err := mgr.Start(); err != nil { + t.Fatalf("Start() error = %v, wantErr %v", err, nil) + } + + select { + case <-certLoaded: + case <-time.After(time.Second): + t.Fatal("onCertLoaded callback was not triggered") + } + + // Test idempotency + if err := mgr.Start(); err != nil { + t.Fatalf("Start() second call error = %v, wantErr %v", err, nil) + } +} + +func TestMockP2PForgeCertMgr_Stop(t *testing.T) { + mgr := mock.NewMockP2PForgeCertMgr(nil) + + if err := mgr.Start(); err != nil { + t.Fatalf("Start() error = %v, wantErr %v", err, nil) + } + + mgr.Stop() + + // Test idempotency + mgr.Stop() +} + +func TestMockP2PForgeCertMgr_TLSConfig(t *testing.T) { + mgr := mock.NewMockP2PForgeCertMgr(nil) + cfg := mgr.TLSConfig() + + if cfg == nil { + t.Fatal("TLSConfig() returned nil") + } + + if len(cfg.Certificates) == 0 { + t.Error("TLSConfig() returned no certificates") + } +} + +func TestMockP2PForgeCertMgr_AddressFactory(t *testing.T) { + mgr := mock.NewMockP2PForgeCertMgr(nil) + factory := mgr.AddressFactory() + + addr, _ := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/0") + addrs := []ma.Multiaddr{addr} + + got := factory(addrs) + if len(got) != 1 || !got[0].Equal(addr) { + t.Errorf("AddressFactory() = %v, want %v", got, addrs) + } +} + +func TestMockP2PForgeCertMgr_SetOnCertLoaded(t *testing.T) { + mgr := mock.NewMockP2PForgeCertMgr(nil) + certLoaded := make(chan struct{}) + + mgr.SetOnCertLoaded(func() { + close(certLoaded) + }) + + if err := mgr.Start(); err != nil { + t.Fatalf("Start() error = %v, wantErr %v", err, nil) + } + + select { + case <-certLoaded: + case <-time.After(time.Second): + t.Fatal("onCertLoaded callback was not triggered after SetOnCertLoaded") + } +} + +func TestMockP2PForgeCertMgr_GetCache(t *testing.T) { + mgr := mock.NewMockP2PForgeCertMgr(nil) + if mgr.GetCache() == nil { + t.Error("GetCache() returned nil") + } +} + +func TestMockConfig(t *testing.T) { + cfg := mock.NewMockConfig() + tlsCfg := cfg.TLSConfig() + if tlsCfg == nil { + t.Fatal("TLSConfig() returned nil") + } + if tlsCfg.MinVersion != tls.VersionTLS12 { + t.Errorf("TLSConfig().MinVersion = %v, want %v", tlsCfg.MinVersion, tls.VersionTLS12) + } +} diff --git a/pkg/p2p/libp2p/mock/testdata/cert.pem b/pkg/p2p/libp2p/mock/testdata/cert.pem new file mode 100644 index 00000000000..942876e1fac --- /dev/null +++ b/pkg/p2p/libp2p/mock/testdata/cert.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDCzCCAfOgAwIBAgIUYmVQGLnuPXQb8yC4X69ll6FQjWQwDQYJKoZIhvcNAQEL +BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MCAXDTI1MDgwMTA3MzA0OFoYDzIxMjUw +NzA4MDczMDQ4WjAUMRIwEAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQC/NbDVAd4OdVMNiK+BjAFyTCIiJssgxhi6rWEgzykV +6lv0Q4/loUCwp1ylh6LJa6yjioGmxXl2CqIALcxRXg/DopJFgfIPPhghSHVE6AvV +PGf+Qw85vXfdjX6qymPsRRz5mMj9N9BidL1HgJjxpGw+7aDwXxMCvxwFN46kyfb9 +fawIIlAJv2gxBoKxf+8AXwimPUrXxG1HziMPNTjQGW+zVRzeK4pTNdt+CXcf4vLM +LKv7eRXYiAL7m/6/UE/SQOXDGxvjkOTWgVX2gcV9tMZOaFJBp/WycsexiajRRxLQ +tl1weQhyBmLA7N494sYpwmzF1feybkkrPeU7Vo7T2gLrAgMBAAGjUzBRMB0GA1Ud +DgQWBBRs/abpJGBaT14jL05Fj9PVbc7T4TAfBgNVHSMEGDAWgBRs/abpJGBaT14j +L05Fj9PVbc7T4TAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQA1 +mhrBSvtlW4mhxQAAEfX/JotieeGuG9nUE75LJ+sW5mRDO+5PZeHAajBdWFGftCm2 +vNONQMVkAt8PSPuZ+6t284HSpWLTgYGTu19m9JFDfw3Dj7HIX95c0xq1WdNSDvpR +kRfoe7yc4B6mCKIU4EjAv0lUs21I2oLIbqQdhJneivutyCYcV/hxt7OJZs92g9HX +S8MGeWRCFhoPoOQ9tvszqzf7AB3uqiRuFxk2DkuZ7yN63/+DGj2kTBkf3Qe+4DyS +7DK5XkPdl/ajTXNoLdgYUw/5hdsbcTjnxRr37eKqoa/MtQxzQjoA5HWYoaDX+Vzp +OwmvrIFPPZ+bU+6+Xs5i +-----END CERTIFICATE----- diff --git a/pkg/p2p/libp2p/mock/testdata/key.pem b/pkg/p2p/libp2p/mock/testdata/key.pem new file mode 100644 index 00000000000..9668c2e083f --- /dev/null +++ b/pkg/p2p/libp2p/mock/testdata/key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC/NbDVAd4OdVMN +iK+BjAFyTCIiJssgxhi6rWEgzykV6lv0Q4/loUCwp1ylh6LJa6yjioGmxXl2CqIA +LcxRXg/DopJFgfIPPhghSHVE6AvVPGf+Qw85vXfdjX6qymPsRRz5mMj9N9BidL1H +gJjxpGw+7aDwXxMCvxwFN46kyfb9fawIIlAJv2gxBoKxf+8AXwimPUrXxG1HziMP +NTjQGW+zVRzeK4pTNdt+CXcf4vLMLKv7eRXYiAL7m/6/UE/SQOXDGxvjkOTWgVX2 +gcV9tMZOaFJBp/WycsexiajRRxLQtl1weQhyBmLA7N494sYpwmzF1feybkkrPeU7 +Vo7T2gLrAgMBAAECggEAAlN0carm2Q7pUAgAVX3d4SvUyhEZNdCNw1MNeqgP0IUi +a22y1uyjjC9i4PdputYh+3bqgS1JdpmB6botbiQY0XZkyGQCt9fAB9yvjZRmylCz +b0mNe23l+lo2lvPimLGxmrTjZUwbfCR1m0Jupqoi4TqOud7nRohxwIQ6Zjmm3Y53 +Ybiv8h46mhy2NZFI2zwaO9q0MpYozCsCR5Xi6DasJZedlAB+185mTJo79e2AllPI +Y3tsQLgrCc2i9TAoEjvJdCxNDuElusabKgROj3DPr2LEkApeX2EdknNKqMIb+NIk +7htDSYjoJuG1ABRB0vBx7s0OGtk+IC/xYwdYB8wgdQKBgQDszNAHHUGbBr2yh8MP +JXqcoxXiDMOjyrrJcTzkUGXP2tCG05u+do8e4Or1K6WXKJGuITeewd6cW/HE2Mlt +sc58b+6+H3hm1dkuAOj+uF6mV8Al2TxVoRrKlAojAzu9QhkH+9GYR+nGqz7wKSnS +6EQETiCfd5LYrK0OH12wOla+JQKBgQDOtpLWroD5Vjf4KJ2qX4un2V2IKyWMcofH +t4qWbD+6F4Kt/ZCxFKviLBwRffGaatp7E1W0kU76oflsrnpRzSzMearlLIIid8nA +ucXCuyJdQPSivsHJJM6MMQ4BzT2stR8JDtJMNip/JUb0pZFS/aG1BCp+nLSoU23m +q7vyUvJHzwKBgQDcaCaY+Jo/+Z5HtiXQy0m80e9kYA0ZP3FsXoIW4N5jAYBmfj/Q +n/nG/AK2ANI4SAKQ2Uoz8q+JSetXFZEnEQDowiatwA0JarKjJyW3MVSn77VhhTmr +WjDdrb1hqXjJR+SUkcccvpLR4ELMtwO+04G7oBytUVbVZqQNKRTDGwnyIQKBgD+t +1KxX05l76wACmxdqGZ6agoq5J/cNLTDkJMhUDomoRnSNAW7bvFuPVRI6Zxw3wJhb +i3J1tQvWq/zD/yCGAT/4VyIERQ6TMk6xq+9iMKLjqLkd5JqvQQXE8tixPkefADGN +JFGf+hVzCVnCS3NyeMdHwkOAyNJ16Qw/aUWsMcDXAoGBAJpIdS7fu7Odxp76ssjy +AUsUFvGLH3rndYUWANy9xmz5FAxt4pZAbtbIYqPXy+4hpXWhqgOmTti8GDqL1W8g +CdYKvjVOHj5GNMQ6QJ3b1MDmUOeSgaN7NIY7SuKlKT54nYkplej9cd/4h1DpjHar +3eEDaJABlw0eDY7F2qXJZzZ3 +-----END PRIVATE KEY----- diff --git a/pkg/p2p/libp2p/protocols_test.go b/pkg/p2p/libp2p/protocols_test.go index 30c9f8504f1..0bec5ba49af 100644 --- a/pkg/p2p/libp2p/protocols_test.go +++ b/pkg/p2p/libp2p/protocols_test.go @@ -18,8 +18,7 @@ import ( libp2pm "github.com/libp2p/go-libp2p" "github.com/libp2p/go-libp2p/core/host" protocol "github.com/libp2p/go-libp2p/core/protocol" - bhost "github.com/libp2p/go-libp2p/p2p/host/basic" - swarmt "github.com/libp2p/go-libp2p/p2p/net/swarm/testing" + ma "github.com/multiformats/go-multiaddr" "github.com/multiformats/go-multistream" ) @@ -398,25 +397,29 @@ func TestPing(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) defer cancel() + listenAddr, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/0") + if err != nil { + t.Fatal(err) + } + + hostFactory := libp2p.WithHostFactory( + func(opts ...libp2pm.Option) (host.Host, error) { + opts = append(opts, libp2pm.ListenAddrs(listenAddr)) + + h, err := libp2pm.New(opts...) + if err != nil { + return nil, err + } + return h, nil + }, + ) + s1, _ := newService(t, 1, libp2pServiceOpts{ - libp2pOpts: libp2p.WithHostFactory( - func(...libp2pm.Option) (host.Host, error) { - return bhost.NewHost(swarmt.GenSwarm(t), &bhost.HostOpts{EnablePing: true}) - }, - ), + libp2pOpts: hostFactory, }) s2, _ := newService(t, 1, libp2pServiceOpts{ - libp2pOpts: libp2p.WithHostFactory( - func(...libp2pm.Option) (host.Host, error) { - host, err := bhost.NewHost(swarmt.GenSwarm(t), &bhost.HostOpts{EnablePing: true}) - if err != nil { - t.Fatalf("start host: %v", err) - } - host.Start() - return host, nil - }, - ), + libp2pOpts: hostFactory, }) addr := serviceUnderlayAddress(t, s1) diff --git a/pkg/p2p/libp2p/static_resolver.go b/pkg/p2p/libp2p/static_resolver.go index c5416862e1b..5927b2b022a 100644 --- a/pkg/p2p/libp2p/static_resolver.go +++ b/pkg/p2p/libp2p/static_resolver.go @@ -5,12 +5,10 @@ package libp2p import ( - "errors" "fmt" "net" "strings" - libp2ppeer "github.com/libp2p/go-libp2p/core/peer" ma "github.com/multiformats/go-multiaddr" ) @@ -40,16 +38,7 @@ func newStaticAddressResolver(addr string, lookupIP func(host string) ([]net.IP, } func (r *staticAddressResolver) Resolve(observedAddress ma.Multiaddr) (ma.Multiaddr, error) { - observableAddrInfo, err := libp2ppeer.AddrInfoFromP2pAddr(observedAddress) - if err != nil { - return nil, err - } - - if len(observableAddrInfo.Addrs) < 1 { - return nil, errors.New("invalid observed address") - } - - observedAddrSplit := strings.Split(observableAddrInfo.Addrs[0].String(), "/") + observedAddrSplit := strings.Split(observedAddress.String(), "/") // if address is not in a form of '/ipversion/ip/protocol/port/...` don't compare to addresses and return it if len(observedAddrSplit) < 5 { @@ -69,12 +58,55 @@ func (r *staticAddressResolver) Resolve(observedAddress ma.Multiaddr) (ma.Multia } else { port = observedAddrSplit[4] } - a, err := ma.NewMultiaddr(multiProto + "/" + observedAddrSplit[3] + "/" + port) + + const ipv4 = "ip4" + const ipv6 = "ip6" + + // replace sni protocol ip subdomain in domain name + otherComponents := observedAddrSplit[5:] + ipSplit := strings.Split(multiProto, "/") + expectedIPSniHost := newIPSniHost(observedAddrSplit[2], observedAddrSplit[1] == ipv4) + if len(ipSplit) == 3 && (ipSplit[1] == ipv4 || ipSplit[1] == ipv6) { + for i, part := range otherComponents { + if part == "sni" && i+1 < len(otherComponents) { + ip := ipSplit[2] + isIPv4 := ipSplit[1] == ipv4 + + sniParts := strings.Split(otherComponents[i+1], ".") + if len(sniParts) > 0 { + if sniParts[0] != expectedIPSniHost { + continue + } + sniParts[0] = newIPSniHost(ip, isIPv4) + otherComponents[i+1] = strings.Join(sniParts, ".") + } + break // assume only one sni component + } + } + } + + a, err := ma.NewMultiaddr(multiProto + "/" + observedAddrSplit[3] + "/" + port + "/" + strings.Join(otherComponents, "/")) if err != nil { return nil, err } - return buildUnderlayAddress(a, observableAddrInfo.ID) + return a, nil +} + +func newIPSniHost(ip string, isIPv4 bool) string { + if isIPv4 { + return strings.ReplaceAll(ip, ".", "-") + } + + // IPv6 + newSniHost := strings.ReplaceAll(ip, ":", "-") + if newSniHost[0] == '-' { + newSniHost = "0" + newSniHost + } + if newSniHost[len(newSniHost)-1] == '-' { + newSniHost = newSniHost + "0" + } + return newSniHost } func getMultiProto(host string, lookupIP func(host string) ([]net.IP, error)) (string, error) { diff --git a/pkg/p2p/libp2p/static_resolver_test.go b/pkg/p2p/libp2p/static_resolver_test.go index c45cce0142f..8e9f022721d 100644 --- a/pkg/p2p/libp2p/static_resolver_test.go +++ b/pkg/p2p/libp2p/static_resolver_test.go @@ -27,6 +27,12 @@ func TestStaticAddressResolver(t *testing.T) { observableAddress: "/ip4/127.0.0.1/tcp/7071/p2p/16Uiu2HAkyyGKpjBiCkVqCKoJa6RzzZw9Nr7hGogsMPcdad1KyMmd", want: "/ip4/127.0.0.1/tcp/30123/p2p/16Uiu2HAkyyGKpjBiCkVqCKoJa6RzzZw9Nr7hGogsMPcdad1KyMmd", }, + { + name: "without p2p protocols", + natAddr: ":30123", + observableAddress: "/ip4/127.0.0.1/tcp/7071", + want: "/ip4/127.0.0.1/tcp/30123", + }, { name: "replace ip v4", natAddr: "192.168.1.34:", @@ -81,10 +87,38 @@ func TestStaticAddressResolver(t *testing.T) { observableAddress: "/ip4/127.0.0.1/tcp/7071/p2p/16Uiu2HAkyyGKpjBiCkVqCKoJa6RzzZw9Nr7hGogsMPcdad1KyMmd", want: "/dns/ipv4and6.com/tcp/30777/p2p/16Uiu2HAkyyGKpjBiCkVqCKoJa6RzzZw9Nr7hGogsMPcdad1KyMmd", }, + { + name: "replace ip and port with complex multiaddr", + natAddr: "192.168.1.34:30777", + observableAddress: "/ip4/10.233.99.40/tcp/1635/tls/sni/*.libp2p.direct/ws/p2p/QmWbXocGMpfa8zApx9kCNwfmc35bbRJv136bdtuQjbR4wL", + want: "/ip4/192.168.1.34/tcp/30777/tls/sni/*.libp2p.direct/ws/p2p/QmWbXocGMpfa8zApx9kCNwfmc35bbRJv136bdtuQjbR4wL", + }, + { + name: "replace ip and port with complex multiaddr without p2p protocol", + natAddr: "192.168.1.34:30777", + observableAddress: "/ip4/10.233.99.40/tcp/1635/tls/sni/*.libp2p.direct/ws", + want: "/ip4/192.168.1.34/tcp/30777/tls/sni/*.libp2p.direct/ws", + }, + { + name: "replace ip and port with complex multiaddr without p2p protocol and with full domain", + natAddr: "192.168.1.34:30777", + observableAddress: "/ip4/10.233.99.40/tcp/1635/tls/sni/10-233-99-40.k2k4r8nnyi7fa2p2t2fjfam2vdyk5esk1iwlfkdz1yezu56o0o0b966w.libp2p.direct/ws", + want: "/ip4/192.168.1.34/tcp/30777/tls/sni/192-168-1-34.k2k4r8nnyi7fa2p2t2fjfam2vdyk5esk1iwlfkdz1yezu56o0o0b966w.libp2p.direct/ws", + }, + { + name: "replace ip v6 and port with complex multiaddr without p2p protocol and with full domain", + natAddr: "[::1]:30777", + observableAddress: "/ip4/10.233.99.40/tcp/1635/tls/sni/10-233-99-40.k2k4r8nnyi7fa2p2t2fjfam2vdyk5esk1iwlfkdz1yezu56o0o0b966w.libp2p.direct/ws", + want: "/ip6/::1/tcp/30777/tls/sni/0--1.k2k4r8nnyi7fa2p2t2fjfam2vdyk5esk1iwlfkdz1yezu56o0o0b966w.libp2p.direct/ws", + }, + { + name: "do not change arbitrary sni domain", + natAddr: "[::1]:30777", + observableAddress: "/ip4/10.233.99.40/tcp/1635/tls/sni/some-node.ethswarm.org/ws", + want: "/ip6/::1/tcp/30777/tls/sni/some-node.ethswarm.org/ws", + }, } { t.Run(tc.name, func(t *testing.T) { - t.Parallel() - r, err := libp2p.NewStaticAddressResolver(tc.natAddr, func(host string) ([]net.IP, error) { hosts := map[string][]net.IP{ "ipv4.com": { diff --git a/pkg/p2p/libp2p/upnp.go b/pkg/p2p/libp2p/upnp.go deleted file mode 100644 index e0dcc3f577e..00000000000 --- a/pkg/p2p/libp2p/upnp.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2020 The Swarm Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package libp2p - -import ( - "errors" - "strings" - - "github.com/libp2p/go-libp2p/core/host" - libp2ppeer "github.com/libp2p/go-libp2p/core/peer" - ma "github.com/multiformats/go-multiaddr" -) - -type UpnpAddressResolver struct { - host host.Host -} - -// Resolve checks if there is a possible better advertisable underlay then the provided observed address. -// In some NAT situations, for example in the case when nodes are behind upnp, observer might send the observed address with a wrong port. -// In this case, observed address is compared to addresses provided by host, and if there is a same address but with different port, that one is used as advertisable address instead of provided observed one. -// TODO: this is a quickfix and it will be improved in the future -func (r *UpnpAddressResolver) Resolve(observedAddress ma.Multiaddr) (ma.Multiaddr, error) { - observableAddrInfo, err := libp2ppeer.AddrInfoFromP2pAddr(observedAddress) - if err != nil { - return nil, err - } - - if len(observableAddrInfo.Addrs) < 1 { - return nil, errors.New("invalid observed address") - } - - observedAddrSplit := strings.Split(observableAddrInfo.Addrs[0].String(), "/") - - // if address is not in a form of '/ipversion/ip/protocol/port/...` don't compare to addresses and return it - if len(observedAddrSplit) < 5 { - return observedAddress, nil - } - - observedAddressPort := observedAddrSplit[4] - - // observervedAddressShort is an obaserved address without port - observervedAddressShort := strings.Join(append(observedAddrSplit[:4], observedAddrSplit[5:]...), "/") - - for _, a := range r.host.Addrs() { - asplit := strings.Split(a.String(), "/") - if len(asplit) != len(observedAddrSplit) { - continue - } - - aport := asplit[4] - if strings.Join(append(asplit[:4], asplit[5:]...), "/") != observervedAddressShort { - continue - } - - if aport != observedAddressPort { - aaddress, err := buildUnderlayAddress(a, observableAddrInfo.ID) - if err != nil { - continue - } - - return aaddress, nil - } - } - - return observedAddress, nil -} diff --git a/pkg/puller/puller.go b/pkg/puller/puller.go index d53a45348e4..0bcdcfae9cb 100644 --- a/pkg/puller/puller.go +++ b/pkg/puller/puller.go @@ -191,9 +191,7 @@ func (p *Puller) manage(ctx context.Context) { // disconnectPeer cancels all existing syncing and removes the peer entry from the syncing map. // Must be called under lock. func (p *Puller) disconnectPeer(addr swarm.Address) { - loggerV2 := p.logger.V(2).Register() - - loggerV2.Debug("disconnecting peer", "peer_address", addr) + p.logger.Debug("disconnecting peer", "peer_address", addr) if peer, ok := p.syncPeers[addr.ByteString()]; ok { peer.mtx.Lock() peer.stop() @@ -300,8 +298,6 @@ func (p *Puller) syncPeer(ctx context.Context, peer *syncPeer, storageRadius uin // syncPeerBin will start historical and live syncing for the peer for a particular bin. // Must be called under syncPeer lock. func (p *Puller) syncPeerBin(parentCtx context.Context, peer *syncPeer, bin uint8, cursor uint64) { - loggerV2 := p.logger.V(2).Register() - ctx, cancel := context.WithCancel(parentCtx) peer.setBinCancel(cancel, bin) @@ -331,14 +327,13 @@ func (p *Puller) syncPeerBin(parentCtx context.Context, peer *syncPeer, bin uint select { case <-ctx.Done(): - loggerV2.Debug("syncWorker context cancelled", "peer_address", address, "bin", bin) + p.logger.Debug("syncWorker context cancelled", "peer_address", address, "bin", bin) return default: } p.metrics.SyncWorkerIterCounter.Inc() - syncStart := time.Now() top, count, err := p.syncer.Sync(ctx, address, bin, start) if top == math.MaxUint64 { @@ -353,7 +348,7 @@ func (p *Puller) syncPeerBin(parentCtx context.Context, peer *syncPeer, bin uint p.logger.Debug("syncWorker interval failed, quitting", "error", err, "peer_address", address, "bin", bin, "cursor", cursor, "start", start, "topmost", top) return } - loggerV2.Debug("syncWorker interval failed", "error", err, "peer_address", address, "bin", bin, "cursor", cursor, "start", start, "topmost", top) + p.logger.Debug("syncWorker interval failed", "error", err, "peer_address", address, "bin", bin, "cursor", cursor, "start", start, "topmost", top) } _ = p.limiter.WaitN(ctx, count) @@ -372,7 +367,6 @@ func (p *Puller) syncPeerBin(parentCtx context.Context, peer *syncPeer, bin uint p.logger.Error(err, "syncWorker could not persist interval for peer, quitting", "peer_address", address) return } - loggerV2.Debug("syncWorker pulled", "bin", bin, "start", start, "topmost", top, "isHistorical", isHistorical, "duration", time.Since(syncStart), "peer_address", address) start = top + 1 } } diff --git a/pkg/pullsync/pullsync.go b/pkg/pullsync/pullsync.go index 064099e765c..436600a2f76 100644 --- a/pkg/pullsync/pullsync.go +++ b/pkg/pullsync/pullsync.go @@ -49,7 +49,6 @@ const ( MaxCursor = math.MaxUint64 DefaultMaxPage uint64 = 250 pageTimeout = time.Second - makeOfferTimeout = 15 * time.Minute handleMaxChunksPerSecond = 250 handleRequestsLimitRate = time.Second / handleMaxChunksPerSecond // handle max `handleMaxChunksPerSecond` chunks per second per peer ) @@ -228,7 +227,7 @@ func (s *Syncer) handler(streamCtx context.Context, p p2p.Peer, stream p2p.Strea // Sync syncs a batch of chunks starting at a start BinID. // It returns the BinID of highest chunk that was synced from the given // batch and the total number of chunks the downstream peer has sent. -func (s *Syncer) Sync(ctx context.Context, peer swarm.Address, bin uint8, start uint64) (uint64, int, error) { +func (s *Syncer) Sync(ctx context.Context, peer swarm.Address, bin uint8, start uint64) (topmost uint64, count int, err error) { stream, err := s.streamer.NewStream(ctx, peer, nil, protocolName, protocolVersion, streamName) if err != nil { @@ -237,7 +236,6 @@ func (s *Syncer) Sync(ctx context.Context, peer swarm.Address, bin uint8, start defer func() { if err != nil { _ = stream.Reset() - s.logger.Debug("error syncing peer", "peer_address", peer, "bin", bin, "start", start, "error", err) } else { stream.FullClose() } @@ -261,7 +259,7 @@ func (s *Syncer) Sync(ctx context.Context, peer swarm.Address, bin uint8, start return offer.Topmost, 0, nil } - topmost := offer.Topmost + topmost = offer.Topmost var ( bvLen = len(offer.Chunks) @@ -403,9 +401,6 @@ func (s *Syncer) Sync(ctx context.Context, peer swarm.Address, bin uint8, start // makeOffer tries to assemble an offer for a given requested interval. func (s *Syncer) makeOffer(ctx context.Context, rn pb.Get) (*pb.Offer, error) { - ctx, cancel := context.WithTimeout(ctx, makeOfferTimeout) - defer cancel() - addrs, top, err := s.collectAddrs(ctx, uint8(rn.Bin), rn.Start) if err != nil { return nil, err @@ -430,8 +425,6 @@ type collectAddrsResult struct { // After the arrival of the first chunk, the subsequent chunks have a limited amount of time to arrive, // after which the function returns the collected slice of chunks. func (s *Syncer) collectAddrs(ctx context.Context, bin uint8, start uint64) ([]*storer.BinC, uint64, error) { - loggerV2 := s.logger.V(2).Register() - v, _, err := s.intervalsSF.Do(ctx, sfKey(bin, start), func(ctx context.Context) (*collectAddrsResult, error) { var ( chs []*storer.BinC @@ -476,7 +469,7 @@ func (s *Syncer) collectAddrs(ctx context.Context, bin uint8, start uint64) ([]* case <-ctx.Done(): return nil, ctx.Err() case <-timerC: - loggerV2.Debug("batch timeout timer triggered") + s.logger.Debug("batch timeout timer triggered") // return batch if new chunks are not received after some time break LOOP } @@ -518,17 +511,14 @@ func (s *Syncer) processWant(ctx context.Context, o *pb.Offer, w *pb.Want) ([]sw } func (s *Syncer) GetCursors(ctx context.Context, peer swarm.Address) (retr []uint64, epoch uint64, err error) { - loggerV2 := s.logger.V(2).Register() - stream, err := s.streamer.NewStream(ctx, peer, nil, protocolName, protocolVersion, cursorStreamName) if err != nil { return nil, 0, fmt.Errorf("new stream: %w", err) } - loggerV2.Debug("getting cursors from peer", "peer_address", peer) + s.logger.Debug("getting cursors from peer", "peer_address", peer) defer func() { if err != nil { _ = stream.Reset() - loggerV2.Debug("error getting cursors from peer", "peer_address", peer, "error", err) } else { stream.FullClose() } @@ -549,14 +539,11 @@ func (s *Syncer) GetCursors(ctx context.Context, peer swarm.Address) (retr []uin } func (s *Syncer) cursorHandler(ctx context.Context, p p2p.Peer, stream p2p.Stream) (err error) { - loggerV2 := s.logger.V(2).Register() - w, r := protobuf.NewWriterAndReader(stream) - loggerV2.Debug("peer wants cursors", "peer_address", p.Address) + s.logger.Debug("peer wants cursors", "peer_address", p.Address) defer func() { if err != nil { _ = stream.Reset() - loggerV2.Debug("error getting cursors for peer", "peer_address", p.Address, "error", err) } else { _ = stream.FullClose() } diff --git a/pkg/pushsync/main_test.go b/pkg/pushsync/main_test.go index 1364235fc42..cfd84a04c51 100644 --- a/pkg/pushsync/main_test.go +++ b/pkg/pushsync/main_test.go @@ -11,5 +11,5 @@ import ( ) func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) + goleak.VerifyTestMain(m, goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start")) } diff --git a/pkg/pushsync/pushsync.go b/pkg/pushsync/pushsync.go index 715680430cf..05c4aef3d41 100644 --- a/pkg/pushsync/pushsync.go +++ b/pkg/pushsync/pushsync.go @@ -186,7 +186,10 @@ func (ps *PushSync) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream) ps.metrics.TotalHandlerTime.WithLabelValues("failure").Observe(time.Since(now).Seconds()) ps.metrics.TotalHandlerErrors.Inc() if !attemptedWrite { - _ = w.WriteMsgWithContext(ctx, &pb.Receipt{Err: err.Error()}) + if writeErr := w.WriteMsgWithContext(ctx, &pb.Receipt{Err: err.Error()}); writeErr == nil { + _ = stream.FullClose() + return + } } _ = stream.Reset() } else { diff --git a/pkg/pushsync/pushsync_integration_test.go b/pkg/pushsync/pushsync_integration_test.go new file mode 100644 index 00000000000..e9b2fa30f11 --- /dev/null +++ b/pkg/pushsync/pushsync_integration_test.go @@ -0,0 +1,201 @@ +// Copyright 2025 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pushsync_test + +import ( + "context" + "crypto/ecdsa" + "encoding/json" + "strings" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + accountingmock "github.com/ethersphere/bee/v2/pkg/accounting/mock" + "github.com/ethersphere/bee/v2/pkg/crypto" + "github.com/ethersphere/bee/v2/pkg/log" + "github.com/ethersphere/bee/v2/pkg/p2p/libp2p/libp2ptest" + postagetesting "github.com/ethersphere/bee/v2/pkg/postage/testing" + pricermock "github.com/ethersphere/bee/v2/pkg/pricer/mock" + "github.com/ethersphere/bee/v2/pkg/pushsync" + "github.com/ethersphere/bee/v2/pkg/soc" + stabilizationmock "github.com/ethersphere/bee/v2/pkg/stabilization/mock" + "github.com/ethersphere/bee/v2/pkg/storage" + "github.com/ethersphere/bee/v2/pkg/storage/inmemchunkstore" + testingc "github.com/ethersphere/bee/v2/pkg/storage/testing" + "github.com/ethersphere/bee/v2/pkg/swarm" + topologymock "github.com/ethersphere/bee/v2/pkg/topology/mock" +) + +// TestPushSyncIntegration is a regression test for the fix of missing +// FullClose on pushsync streams. +func TestPushSyncIntegration(t *testing.T) { + // Capture logs to check for errors + var logMessages libp2ptest.SafeBuffer + + // Enable debug logging + logger := log.NewLogger("test", + log.WithVerbosity(log.VerbosityDebug), + log.WithJSONOutput(), + log.WithSink(&logMessages), + ) + + // Create two libp2p services + server, serverAddr := libp2ptest.NewLibp2pService(t, 1, logger.WithValues("libp2p", "server").Build()) + client, clientAddr := libp2ptest.NewLibp2pService(t, 1, logger.WithValues("libp2p", "client").Build()) + + // Server setup + serverStorer := &testIntegrationStorer{ChunkStore: inmemchunkstore.New()} + serverSigner := crypto.NewDefaultSigner(mustGenerateKey(t)) + + serverPushSync := pushsync.New( + serverAddr, + 1, + common.HexToHash("0x1").Bytes(), + server, // Use libp2p service as streamer + serverStorer, + func() (uint8, error) { return 0, nil }, + topologymock.NewTopologyDriver(), // No peers needed for server in this test + true, + func(swarm.Chunk) {}, // unwrap + func(*soc.SOC) {}, // gsocHandler + func(c swarm.Chunk) (swarm.Chunk, error) { + return c.WithStamp(postagetesting.MustNewValidStamp(serverSigner, c.Address())), nil + }, + logger.WithValues("pushsync", "server").Build(), + accountingmock.NewAccounting(), + pricermock.NewMockService(10, 10), + serverSigner, + nil, + stabilizationmock.NewSubscriber(true), + 0, + ) + t.Cleanup(func() { serverPushSync.Close() }) + + // Register pushsync protocol on server + if err := server.AddProtocol(serverPushSync.Protocol()); err != nil { + t.Fatal(err) + } + + // Client setup + clientStorer := &testIntegrationStorer{ChunkStore: inmemchunkstore.New()} + clientSigner := crypto.NewDefaultSigner(mustGenerateKey(t)) + + // Client needs to know server is the closest peer + clientTopology := topologymock.NewTopologyDriver(topologymock.WithPeers(serverAddr)) + + clientPushSync := pushsync.New( + clientAddr, + 1, + common.HexToHash("0x1").Bytes(), + client, // Use libp2p service as streamer + clientStorer, + func() (uint8, error) { return 0, nil }, + clientTopology, + true, + func(swarm.Chunk) {}, // unwrap + func(*soc.SOC) {}, // gsocHandler + func(c swarm.Chunk) (swarm.Chunk, error) { + return c.WithStamp(postagetesting.MustNewValidStamp(clientSigner, c.Address())), nil + }, + logger.WithValues("pushsync", "client").Build(), + accountingmock.NewAccounting(), + pricermock.NewMockService(10, 10), + clientSigner, + nil, + stabilizationmock.NewSubscriber(true), + 0, + ) + t.Cleanup(func() { clientPushSync.Close() }) + + // Register pushsync protocol on client + if err := client.AddProtocol(clientPushSync.Protocol()); err != nil { + t.Fatal(err) + } + + // Connect client to server + serverAddrs, err := server.Addresses() + if err != nil { + t.Fatal(err) + } + + _, err = client.Connect(context.Background(), serverAddrs) + if err != nil { + t.Fatal(err) + } + + // Push chunk + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + // Generate a chunk that is closer to the server than the client + var chunk swarm.Chunk + for { + chunk = testingc.GenerateTestRandomChunk() + // Check if chunk is closer to server + if swarm.Proximity(serverAddr.Bytes(), chunk.Address().Bytes()) > swarm.Proximity(clientAddr.Bytes(), chunk.Address().Bytes()) { + break + } + } + + _, err = clientPushSync.PushChunkToClosest(ctx, chunk) + if err != nil { + t.Fatal(err) + } + + // Verify chunk is stored on server + has, err := serverStorer.Has(ctx, chunk.Address()) + if err != nil { + t.Fatal(err) + } + if !has { + t.Fatal("chunk not found on server") + } + + // Validate that streams are properly closed by the pushsync service + // by checking for "stream reset" log entries + streamResetLogLine := "" + for line := range strings.SplitSeq(logMessages.String(), "\n") { + if line == "" { + continue + } + var entry map[string]any + if err := json.Unmarshal([]byte(line), &entry); err != nil { + t.Fatalf("failed to unmarshal log entry: %v", err) + } + errorString, ok := entry["error"].(string) + if !ok { + continue + } + if strings.Contains(errorString, "stream reset") { + streamResetLogLine = line + break + } + } + if streamResetLogLine != "" { + t.Errorf("found stream reset log line: %s", streamResetLogLine) + } +} + +type testIntegrationStorer struct { + storage.ChunkStore +} + +func (t *testIntegrationStorer) Report(context.Context, swarm.Chunk, storage.ChunkState) error { + return nil +} + +func (t *testIntegrationStorer) ReservePutter() storage.Putter { + return t.ChunkStore +} + +func mustGenerateKey(t *testing.T) *ecdsa.PrivateKey { + t.Helper() + k, err := crypto.GenerateSecp256k1Key() + if err != nil { + t.Fatal(err) + } + return k +} diff --git a/pkg/retrieval/main_test.go b/pkg/retrieval/main_test.go index 8be3983b65d..532dcc95bcb 100644 --- a/pkg/retrieval/main_test.go +++ b/pkg/retrieval/main_test.go @@ -16,5 +16,6 @@ func TestMain(m *testing.M) { // pkg/p2p package has some leak issues, we ignore them here as they are not in current scope goleak.IgnoreTopFunction("github.com/ethersphere/bee/v2/pkg/p2p/protobuf.Reader.ReadMsgWithContext"), goleak.IgnoreTopFunction("github.com/ethersphere/bee/v2/pkg/p2p/streamtest.(*record).Read"), + goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), ) } diff --git a/pkg/retrieval/retrieval.go b/pkg/retrieval/retrieval.go index 2fce97be5a2..f1e5f5193b7 100644 --- a/pkg/retrieval/retrieval.go +++ b/pkg/retrieval/retrieval.go @@ -431,7 +431,10 @@ func (s *Service) handler(p2pctx context.Context, p p2p.Peer, stream p2p.Stream) defer func() { if err != nil { if !attemptedWrite { - _ = w.WriteMsgWithContext(ctx, &pb.Delivery{Err: err.Error()}) + if writeErr := w.WriteMsgWithContext(ctx, &pb.Delivery{Err: err.Error()}); writeErr == nil { + _ = stream.FullClose() + return + } } _ = stream.Reset() } else { diff --git a/pkg/retrieval/retrieval_integration_test.go b/pkg/retrieval/retrieval_integration_test.go new file mode 100644 index 00000000000..de37e46018a --- /dev/null +++ b/pkg/retrieval/retrieval_integration_test.go @@ -0,0 +1,153 @@ +// Copyright 2025 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package retrieval_test + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" + + accountingmock "github.com/ethersphere/bee/v2/pkg/accounting/mock" + "github.com/ethersphere/bee/v2/pkg/log" + "github.com/ethersphere/bee/v2/pkg/p2p/libp2p/libp2ptest" + pricermock "github.com/ethersphere/bee/v2/pkg/pricer/mock" + "github.com/ethersphere/bee/v2/pkg/retrieval" + "github.com/ethersphere/bee/v2/pkg/storage/inmemchunkstore" + testingc "github.com/ethersphere/bee/v2/pkg/storage/testing" + "github.com/ethersphere/bee/v2/pkg/swarm" + "github.com/ethersphere/bee/v2/pkg/topology" + topologymock "github.com/ethersphere/bee/v2/pkg/topology/mock" +) + +// TestRetrievalIntegration is a regression test for the fix of missing +// FullClose on retrieval streams. +func TestRetrievalIntegration(t *testing.T) { + // Capture logs to check for errors + var logMessages libp2ptest.SafeBuffer + + // Enable debug logging + logger := log.NewLogger("test", + log.WithVerbosity(log.VerbosityDebug), + log.WithJSONOutput(), + log.WithSink(&logMessages), + ) + + // Create two libp2p services + server, serverAddr := libp2ptest.NewLibp2pService(t, 1, logger.WithValues("libp2p", "server").Build()) + client, clientAddr := libp2ptest.NewLibp2pService(t, 1, logger.WithValues("libp2p", "client").Build()) + + // Setup chunk + chunk := testingc.FixtureChunk("0033") + + // Server setup + serverStorer := &testStorer{ChunkStore: inmemchunkstore.New()} + err := serverStorer.Put(context.Background(), chunk) + if err != nil { + t.Fatal(err) + } + + serverRetrieval := retrieval.New( + serverAddr, + func() (uint8, error) { return swarm.MaxBins, nil }, + serverStorer, + server, // Use libp2p service as streamer + topologymock.NewTopologyDriver(), // No peers needed for server in this test + logger.WithValues("retrieval", "server").Build(), + accountingmock.NewAccounting(), + pricermock.NewMockService(10, 10), + nil, + false, + ) + t.Cleanup(func() { serverRetrieval.Close() }) + + // Register retrieval protocol on server + if err := server.AddProtocol(serverRetrieval.Protocol()); err != nil { + t.Fatal(err) + } + + // Client setup + clientStorer := &testStorer{ChunkStore: inmemchunkstore.New()} + + // Client needs to know server is the closest peer + clientTopology := topologymock.NewTopologyDriver(topologymock.WithPeers(serverAddr)) + + clientRetrieval := retrieval.New( + clientAddr, + func() (uint8, error) { return swarm.MaxBins, nil }, + clientStorer, + client, // Use libp2p service as streamer + clientTopology, + logger.WithValues("retrieval", "client").Build(), + accountingmock.NewAccounting(), + pricermock.NewMockService(10, 10), + nil, + false, + ) + t.Cleanup(func() { clientRetrieval.Close() }) + + // Register retrieval protocol on client + if err := client.AddProtocol(clientRetrieval.Protocol()); err != nil { + t.Fatal(err) + } + + // Connect client to server + serverAddrs, err := server.Addresses() + if err != nil { + t.Fatal(err) + } + + _, err = client.Connect(context.Background(), serverAddrs) + if err != nil { + t.Fatal(err) + } + + // Retrieve chunk + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + got, err := clientRetrieval.RetrieveChunk(ctx, chunk.Address(), swarm.ZeroAddress) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(got.Data(), chunk.Data()) { + t.Fatalf("request and response data not equal. got %s want %s", got.Data(), chunk.Data()) + } + + // Retrieve non-existing chunk + nonExistingChunk := testingc.GenerateTestRandomChunk() + _, err = clientRetrieval.RetrieveChunk(ctx, nonExistingChunk.Address(), swarm.ZeroAddress) + if !errors.Is(err, topology.ErrNotFound) { + t.Fatalf("expected error retrieving non-existing chunk, got %v", err) + } + + // Validate that streams are properly closed by the retrieval service + // by checking for "stream reset" log entries + streamResetLogLine := "" + for line := range strings.SplitSeq(logMessages.String(), "\n") { + if line == "" { + continue + } + var entry map[string]any + if err := json.Unmarshal([]byte(line), &entry); err != nil { + t.Fatalf("failed to unmarshal log entry: %v", err) + } + errorString, ok := entry["error"].(string) + if !ok { + continue + } + if strings.Contains(errorString, "stream reset") { + streamResetLogLine = line + break + } + } + if streamResetLogLine != "" { + t.Errorf("found stream reset log line: %s", streamResetLogLine) + } +} diff --git a/pkg/settlement/swap/swapprotocol/swapprotocol.go b/pkg/settlement/swap/swapprotocol/swapprotocol.go index 45715621208..5153a307bc3 100644 --- a/pkg/settlement/swap/swapprotocol/swapprotocol.go +++ b/pkg/settlement/swap/swapprotocol/swapprotocol.go @@ -126,7 +126,7 @@ func (s *Service) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream) (e exchangeRate, deduction, err := swap.ParseSettlementResponseHeaders(responseHeaders) if err != nil { if !errors.Is(err, swap.ErrNoDeductionHeader) { - return err + return fmt.Errorf("parse settlement response headers: %w", err) } deduction = big.NewInt(0) } @@ -134,11 +134,15 @@ func (s *Service) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream) (e var signedCheque *chequebook.SignedCheque err = json.Unmarshal(req.Cheque, &signedCheque) if err != nil { - return err + return fmt.Errorf("unmarshal cheque: %w", err) } // signature validation - return s.swap.ReceiveCheque(ctx, p.Address, signedCheque, exchangeRate, deduction) + if err := s.swap.ReceiveCheque(ctx, p.Address, signedCheque, exchangeRate, deduction); err != nil { + return fmt.Errorf("receive cheque: %w", err) + } + + return nil } func (s *Service) headler(receivedHeaders p2p.Headers, peerAddress swarm.Address) (returnHeaders p2p.Headers) { @@ -163,14 +167,12 @@ func (s *Service) headler(receivedHeaders p2p.Headers, peerAddress swarm.Address // InitiateCheque attempts to send a cheque to a peer. func (s *Service) EmitCheque(ctx context.Context, peer swarm.Address, beneficiary common.Address, amount *big.Int, issue IssueFunc) (balance *big.Int, err error) { - loggerV1 := s.logger.V(1).Register() - ctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() stream, err := s.streamer.NewStream(ctx, peer, nil, protocolName, protocolVersion, streamName) if err != nil { - return nil, err + return nil, fmt.Errorf("new stream: %w", err) } defer func() { if err != nil { @@ -185,7 +187,7 @@ func (s *Service) EmitCheque(ctx context.Context, peer swarm.Address, beneficiar exchangeRate, deduction, err := swap.ParseSettlementResponseHeaders(returnedHeaders) if err != nil { if !errors.Is(err, swap.ErrNoDeductionHeader) { - return nil, err + return nil, fmt.Errorf("parse settlement response headers: %w", err) } deduction = big.NewInt(0) } @@ -195,7 +197,7 @@ func (s *Service) EmitCheque(ctx context.Context, peer swarm.Address, beneficiar // get whether peer have deducted in the past checkPeer, err := s.swap.GetDeductionByPeer(peer) if err != nil { - return nil, err + return nil, fmt.Errorf("get deduction by peer: %w", err) } // if peer is not entitled for deduction but sent non zero deduction value, return with error @@ -206,7 +208,7 @@ func (s *Service) EmitCheque(ctx context.Context, peer swarm.Address, beneficiar // get current global exchangeRate rate and deduction checkExchangeRate, checkDeduction, err := s.priceOracle.CurrentRates() if err != nil { - return nil, err + return nil, fmt.Errorf("get current rates: %w", err) } // exchangeRate rates should match @@ -232,7 +234,7 @@ func (s *Service) EmitCheque(ctx context.Context, peer swarm.Address, beneficiar } // sending cheque - loggerV1.Debug("sending cheque message to peer", "peer_address", peer, "cheque", cheque) + s.logger.Debug("sending cheque message to peer", "peer_address", peer, "cheque", cheque) w := protobuf.NewWriter(stream) return w.WriteMsgWithContext(ctx, &pb.EmitCheque{ @@ -241,13 +243,13 @@ func (s *Service) EmitCheque(ctx context.Context, peer swarm.Address, beneficiar }) if err != nil { - return nil, err + return nil, fmt.Errorf("call issue function: %w", err) } if deduction.Cmp(big.NewInt(0)) != 0 { err = s.swap.AddDeductionByPeer(peer) if err != nil { - return nil, err + return nil, fmt.Errorf("add deduction for peer: %w", err) } } diff --git a/pkg/topology/kademlia/kademlia.go b/pkg/topology/kademlia/kademlia.go index 4761b15519a..b5bff41fc9a 100644 --- a/pkg/topology/kademlia/kademlia.go +++ b/pkg/topology/kademlia/kademlia.go @@ -1115,8 +1115,17 @@ func (k *Kad) AnnounceTo(ctx context.Context, addressee, peer swarm.Address, ful // This does not guarantee that a connection will immediately // be made to the peer. func (k *Kad) AddPeers(addrs ...swarm.Address) { - k.knownPeers.Add(addrs...) - k.notifyManageLoop() + toAdd := make([]swarm.Address, 0, len(addrs)) + for _, addr := range addrs { + if !addr.Equal(k.base) { + toAdd = append(toAdd, addr) + } + } + + if len(toAdd) > 0 { + k.knownPeers.Add(toAdd...) + k.notifyManageLoop() + } } func (k *Kad) Pick(peer p2p.Peer) bool { diff --git a/pkg/topology/kademlia/kademlia_test.go b/pkg/topology/kademlia/kademlia_test.go index 4d534ddade8..6750ebe2284 100644 --- a/pkg/topology/kademlia/kademlia_test.go +++ b/pkg/topology/kademlia/kademlia_test.go @@ -2253,3 +2253,72 @@ func generateMultipleUnderlays(t *testing.T, n int, baseUnderlay string) []ma.Mu } return underlays } + +// TestAddPeersSkipsSelf verifies that AddPeers does not add self address +// to the known peers list, preventing self-connection attempts. +func TestAddPeersSkipsSelf(t *testing.T) { + t.Parallel() + + var ( + conns int32 + base, kad, ab, _, signer = newTestKademlia(t, &conns, nil, kademlia.Options{ + ExcludeFunc: defaultExcludeFunc, + }) + ) + + if err := kad.Start(context.Background()); err != nil { + t.Fatal(err) + } + testutil.CleanupCloser(t, kad) + + // Add some regular peers first + peer1 := swarm.RandAddressAt(t, base, 1) + peer2 := swarm.RandAddressAt(t, base, 2) + + addOne(t, signer, kad, ab, peer1) + addOne(t, signer, kad, ab, peer2) + + waitCounter(t, &conns, 2) + + // Now try to add self address along with another peer + peer3 := swarm.RandAddressAt(t, base, 3) + + // Prepare address for peer3 + multiaddr, err := ma.NewMultiaddr(underlayBase + peer3.String()) + if err != nil { + t.Fatal(err) + } + bzzAddr, err := bzz.NewAddress(signer, []ma.Multiaddr{multiaddr}, peer3, 0, nil) + if err != nil { + t.Fatal(err) + } + if err := ab.Put(peer3, *bzzAddr); err != nil { + t.Fatal(err) + } + + // Try to add self (base) and peer3 together + kad.AddPeers(base, peer3) + + // Only peer3 should result in a connection attempt, not base + waitCounter(t, &conns, 1) + + // Verify we have exactly 3 connected peers (peer1, peer2, peer3), not 4 + waitPeers(t, kad, 3) + + // Verify base is not in the connected peers list + foundSelf := false + err = kad.EachConnectedPeer(func(addr swarm.Address, _ uint8) (bool, bool, error) { + if addr.Equal(base) { + foundSelf = true + return true, false, nil + } + return false, false, nil + }, topology.Select{}) + if err != nil { + t.Fatal(err) + } + + if foundSelf { + t.Fatal("self address should not be in connected peers") + } +} From 5857dadc03a4d9f4f05f95b19ed2e860ad9e01fa Mon Sep 17 00:00:00 2001 From: Akrem Chabchoub <121046693+akrem-chabchoub@users.noreply.github.com> Date: Mon, 19 Jan 2026 11:45:07 +0100 Subject: [PATCH 69/96] fix(kademlia): update misleading debug log message for bootnode connection failure (#5320) --- pkg/topology/kademlia/kademlia.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/topology/kademlia/kademlia.go b/pkg/topology/kademlia/kademlia.go index b5bff41fc9a..b31dd6d6c6d 100644 --- a/pkg/topology/kademlia/kademlia.go +++ b/pkg/topology/kademlia/kademlia.go @@ -827,11 +827,10 @@ func (k *Kad) connectBootNodes(ctx context.Context) { if err != nil { if !errors.Is(err, p2p.ErrAlreadyConnected) { - k.logger.Debug("connect to bootnode failed", "bootnode_address", addr, "error", err) - k.logger.Warning("connect to bootnode failed", "bootnode_address", addr) + k.logger.Error(err, "connect to bootnode failed", "bootnode_address", addr) return false, err } - k.logger.Debug("connect to bootnode failed", "bootnode_address", addr, "error", err) + k.logger.Debug("bootnode already connected", "bootnode_address", addr) return false, nil } From c01e11c9c3eaa22c6e06c5d00475ca00a40a9465 Mon Sep 17 00:00:00 2001 From: Akrem Chabchoub <121046693+akrem-chabchoub@users.noreply.github.com> Date: Mon, 19 Jan 2026 18:50:44 +0100 Subject: [PATCH 70/96] test: integrate synctest in access control pkg (#5296) --- pkg/accesscontrol/controller_test.go | 79 +++++++++++++++------------- 1 file changed, 41 insertions(+), 38 deletions(-) diff --git a/pkg/accesscontrol/controller_test.go b/pkg/accesscontrol/controller_test.go index 1fa3da681ce..c282ef825fb 100644 --- a/pkg/accesscontrol/controller_test.go +++ b/pkg/accesscontrol/controller_test.go @@ -9,6 +9,7 @@ import ( "crypto/ecdsa" "reflect" "testing" + "testing/synctest" "time" "github.com/ethersphere/bee/v2/pkg/accesscontrol" @@ -235,44 +236,46 @@ func TestController_UpdateHandler(t *testing.T) { assert.Len(t, gl.Get(), 2) }) t.Run("add and revoke then get from history", func(t *testing.T) { - addRevokeList := []*ecdsa.PublicKey{&grantee.PublicKey} - ref := swarm.RandAddress(t) - _, hRef, encRef, err := c.UploadHandler(ctx, ls, ref, &publisher.PublicKey, swarm.ZeroAddress) - require.NoError(t, err) - - // Need to wait a second before each update call so that a new history mantaray fork is created for the new key(timestamp) entry - time.Sleep(1 * time.Second) - beforeRevokeTS := time.Now().Unix() - _, egranteeRef, hrefUpdate1, _, err := c.UpdateHandler(ctx, ls, gls, swarm.ZeroAddress, hRef, &publisher.PublicKey, addRevokeList, nil) - require.NoError(t, err) - - time.Sleep(1 * time.Second) - granteeRef, _, hrefUpdate2, _, err := c.UpdateHandler(ctx, ls, gls, egranteeRef, hrefUpdate1, &publisher.PublicKey, nil, addRevokeList) - require.NoError(t, err) - - gl, err := accesscontrol.NewGranteeListReference(ctx, ls, granteeRef) - require.NoError(t, err) - assert.Empty(t, gl.Get()) - // expect history reference to be different after grantee list update - assert.NotEqual(t, hrefUpdate1, hrefUpdate2) - - granteeDH := accesscontrol.NewDefaultSession(grantee) - granteeAl := accesscontrol.NewLogic(granteeDH) - granteeCtrl := accesscontrol.NewController(granteeAl) - // download with grantee shall still work with the timestamp before the revoke - decRef, err := granteeCtrl.DownloadHandler(ctx, ls, encRef, &publisher.PublicKey, hrefUpdate2, beforeRevokeTS) - require.NoError(t, err) - assert.Equal(t, ref, decRef) - - // download with grantee shall NOT work with the latest timestamp - decRef, err = granteeCtrl.DownloadHandler(ctx, ls, encRef, &publisher.PublicKey, hrefUpdate2, time.Now().Unix()) - require.Error(t, err) - assert.Equal(t, swarm.ZeroAddress, decRef) - - // publisher shall still be able to download with the timestamp before the revoke - decRef, err = c.DownloadHandler(ctx, ls, encRef, &publisher.PublicKey, hrefUpdate2, beforeRevokeTS) - require.NoError(t, err) - assert.Equal(t, ref, decRef) + synctest.Test(t, func(t *testing.T) { + addRevokeList := []*ecdsa.PublicKey{&grantee.PublicKey} + ref := swarm.RandAddress(t) + _, hRef, encRef, err := c.UploadHandler(ctx, ls, ref, &publisher.PublicKey, swarm.ZeroAddress) + require.NoError(t, err) + + // Need to wait a second before each update call so that a new history mantaray fork is created for the new key(timestamp) entry + time.Sleep(1 * time.Second) + beforeRevokeTS := time.Now().Unix() + _, egranteeRef, hrefUpdate1, _, err := c.UpdateHandler(ctx, ls, gls, swarm.ZeroAddress, hRef, &publisher.PublicKey, addRevokeList, nil) + require.NoError(t, err) + + time.Sleep(1 * time.Second) + granteeRef, _, hrefUpdate2, _, err := c.UpdateHandler(ctx, ls, gls, egranteeRef, hrefUpdate1, &publisher.PublicKey, nil, addRevokeList) + require.NoError(t, err) + + gl, err := accesscontrol.NewGranteeListReference(ctx, ls, granteeRef) + require.NoError(t, err) + assert.Empty(t, gl.Get()) + // expect history reference to be different after grantee list update + assert.NotEqual(t, hrefUpdate1, hrefUpdate2) + + granteeDH := accesscontrol.NewDefaultSession(grantee) + granteeAl := accesscontrol.NewLogic(granteeDH) + granteeCtrl := accesscontrol.NewController(granteeAl) + // download with grantee shall still work with the timestamp before the revoke + decRef, err := granteeCtrl.DownloadHandler(ctx, ls, encRef, &publisher.PublicKey, hrefUpdate2, beforeRevokeTS) + require.NoError(t, err) + assert.Equal(t, ref, decRef) + + // download with grantee shall NOT work with the latest timestamp + decRef, err = granteeCtrl.DownloadHandler(ctx, ls, encRef, &publisher.PublicKey, hrefUpdate2, time.Now().Unix()) + require.Error(t, err) + assert.Equal(t, swarm.ZeroAddress, decRef) + + // publisher shall still be able to download with the timestamp before the revoke + decRef, err = c.DownloadHandler(ctx, ls, encRef, &publisher.PublicKey, hrefUpdate2, beforeRevokeTS) + require.NoError(t, err) + assert.Equal(t, ref, decRef) + }) }) t.Run("add twice", func(t *testing.T) { addList := []*ecdsa.PublicKey{&grantee.PublicKey, &grantee.PublicKey} From c00d65d40d7729f76eb0c4aba91f90bd6fa23627 Mon Sep 17 00:00:00 2001 From: Akrem Chabchoub <121046693+akrem-chabchoub@users.noreply.github.com> Date: Tue, 20 Jan 2026 09:32:35 +0100 Subject: [PATCH 71/96] test(joiner): add synctest to joiner tests (#5265) --- pkg/file/joiner/joiner_test.go | 660 +++++++++++++++++---------------- 1 file changed, 333 insertions(+), 327 deletions(-) diff --git a/pkg/file/joiner/joiner_test.go b/pkg/file/joiner/joiner_test.go index 10035aa2220..0cb81314582 100644 --- a/pkg/file/joiner/joiner_test.go +++ b/pkg/file/joiner/joiner_test.go @@ -15,6 +15,7 @@ import ( mrand "math/rand" "sync" "testing" + "testing/synctest" "time" "github.com/ethersphere/bee/v2/pkg/cac" @@ -639,265 +640,266 @@ func TestPrefetch(t *testing.T) { } func TestJoinerReadAt(t *testing.T) { - t.Parallel() - - store := inmemchunkstore.New() + synctest.Test(t, func(t *testing.T) { + store := inmemchunkstore.New() - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() + ctx, cancel := context.WithTimeout(t.Context(), time.Second) + defer cancel() - // create root chunk with 2 references and the referenced data chunks - rootChunk := filetest.GenerateTestRandomFileChunk(swarm.ZeroAddress, swarm.ChunkSize*2, swarm.SectionSize*2) - err := store.Put(ctx, rootChunk) - if err != nil { - t.Fatal(err) - } + // create root chunk with 2 references and the referenced data chunks + rootChunk := filetest.GenerateTestRandomFileChunk(swarm.ZeroAddress, swarm.ChunkSize*2, swarm.SectionSize*2) + err := store.Put(ctx, rootChunk) + if err != nil { + t.Fatal(err) + } - firstAddress := swarm.NewAddress(rootChunk.Data()[8 : swarm.SectionSize+8]) - firstChunk := filetest.GenerateTestRandomFileChunk(firstAddress, swarm.ChunkSize, swarm.ChunkSize) - err = store.Put(ctx, firstChunk) - if err != nil { - t.Fatal(err) - } + firstAddress := swarm.NewAddress(rootChunk.Data()[8 : swarm.SectionSize+8]) + firstChunk := filetest.GenerateTestRandomFileChunk(firstAddress, swarm.ChunkSize, swarm.ChunkSize) + err = store.Put(ctx, firstChunk) + if err != nil { + t.Fatal(err) + } - secondAddress := swarm.NewAddress(rootChunk.Data()[swarm.SectionSize+8:]) - secondChunk := filetest.GenerateTestRandomFileChunk(secondAddress, swarm.ChunkSize, swarm.ChunkSize) - err = store.Put(ctx, secondChunk) - if err != nil { - t.Fatal(err) - } + secondAddress := swarm.NewAddress(rootChunk.Data()[swarm.SectionSize+8:]) + secondChunk := filetest.GenerateTestRandomFileChunk(secondAddress, swarm.ChunkSize, swarm.ChunkSize) + err = store.Put(ctx, secondChunk) + if err != nil { + t.Fatal(err) + } - j, _, err := joiner.New(ctx, store, store, rootChunk.Address(), redundancy.DefaultLevel) - if err != nil { - t.Fatal(err) - } + j, _, err := joiner.New(ctx, store, store, rootChunk.Address(), redundancy.DefaultLevel) + if err != nil { + t.Fatal(err) + } - b := make([]byte, swarm.ChunkSize) - _, err = j.ReadAt(b, swarm.ChunkSize) - if err != nil { - t.Fatal(err) - } + b := make([]byte, swarm.ChunkSize) + _, err = j.ReadAt(b, swarm.ChunkSize) + if err != nil { + t.Fatal(err) + } - if !bytes.Equal(b, secondChunk.Data()[8:]) { - t.Fatal("data read at offset not equal to expected chunk") - } + if !bytes.Equal(b, secondChunk.Data()[8:]) { + t.Fatal("data read at offset not equal to expected chunk") + } + }) } // TestJoinerOneLevel tests the retrieval of two data chunks immediately -// below the root chunk level. +// below the root chunk level. It verifies that the joiner correctly reads +// and returns data from both chunks in sequence and handles EOF properly. func TestJoinerOneLevel(t *testing.T) { - t.Parallel() + synctest.Test(t, func(t *testing.T) { + store := inmemchunkstore.New() - store := inmemchunkstore.New() - - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() + ctx, cancel := context.WithTimeout(t.Context(), time.Second) + defer cancel() - // create root chunk with 2 references and the referenced data chunks - rootChunk := filetest.GenerateTestRandomFileChunk(swarm.ZeroAddress, swarm.ChunkSize*2, swarm.SectionSize*2) - err := store.Put(ctx, rootChunk) - if err != nil { - t.Fatal(err) - } + // create root chunk with 2 references and the referenced data chunks + rootChunk := filetest.GenerateTestRandomFileChunk(swarm.ZeroAddress, swarm.ChunkSize*2, swarm.SectionSize*2) + err := store.Put(ctx, rootChunk) + if err != nil { + t.Fatal(err) + } - firstAddress := swarm.NewAddress(rootChunk.Data()[8 : swarm.SectionSize+8]) - firstChunk := filetest.GenerateTestRandomFileChunk(firstAddress, swarm.ChunkSize, swarm.ChunkSize) - err = store.Put(ctx, firstChunk) - if err != nil { - t.Fatal(err) - } + firstAddress := swarm.NewAddress(rootChunk.Data()[8 : swarm.SectionSize+8]) + firstChunk := filetest.GenerateTestRandomFileChunk(firstAddress, swarm.ChunkSize, swarm.ChunkSize) + err = store.Put(ctx, firstChunk) + if err != nil { + t.Fatal(err) + } - secondAddress := swarm.NewAddress(rootChunk.Data()[swarm.SectionSize+8:]) - secondChunk := filetest.GenerateTestRandomFileChunk(secondAddress, swarm.ChunkSize, swarm.ChunkSize) - err = store.Put(ctx, secondChunk) - if err != nil { - t.Fatal(err) - } + secondAddress := swarm.NewAddress(rootChunk.Data()[swarm.SectionSize+8:]) + secondChunk := filetest.GenerateTestRandomFileChunk(secondAddress, swarm.ChunkSize, swarm.ChunkSize) + err = store.Put(ctx, secondChunk) + if err != nil { + t.Fatal(err) + } - j, _, err := joiner.New(ctx, store, store, rootChunk.Address(), redundancy.DefaultLevel) - if err != nil { - t.Fatal(err) - } + j, _, err := joiner.New(ctx, store, store, rootChunk.Address(), redundancy.DefaultLevel) + if err != nil { + t.Fatal(err) + } - // verify first chunk content - outBuffer := make([]byte, swarm.ChunkSize) - c, err := j.Read(outBuffer) - if err != nil { - t.Fatal(err) - } - if c != swarm.ChunkSize { - t.Fatalf("expected firstchunk read count %d, got %d", swarm.ChunkSize, c) - } - if !bytes.Equal(outBuffer, firstChunk.Data()[8:]) { - t.Fatalf("firstchunk data mismatch, expected %x, got %x", outBuffer, firstChunk.Data()[8:]) - } + // verify first chunk content + outBuffer := make([]byte, swarm.ChunkSize) + c, err := j.Read(outBuffer) + if err != nil { + t.Fatal(err) + } + if c != swarm.ChunkSize { + t.Fatalf("expected firstchunk read count %d, got %d", swarm.ChunkSize, c) + } + if !bytes.Equal(outBuffer, firstChunk.Data()[8:]) { + t.Fatalf("firstchunk data mismatch, expected %x, got %x", outBuffer, firstChunk.Data()[8:]) + } - // verify second chunk content - c, err = j.Read(outBuffer) - if err != nil { - t.Fatal(err) - } - if c != swarm.ChunkSize { - t.Fatalf("expected secondchunk read count %d, got %d", swarm.ChunkSize, c) - } - if !bytes.Equal(outBuffer, secondChunk.Data()[8:]) { - t.Fatalf("secondchunk data mismatch, expected %x, got %x", outBuffer, secondChunk.Data()[8:]) - } + // verify second chunk content + c, err = j.Read(outBuffer) + if err != nil { + t.Fatal(err) + } + if c != swarm.ChunkSize { + t.Fatalf("expected secondchunk read count %d, got %d", swarm.ChunkSize, c) + } + if !bytes.Equal(outBuffer, secondChunk.Data()[8:]) { + t.Fatalf("secondchunk data mismatch, expected %x, got %x", outBuffer, secondChunk.Data()[8:]) + } - // verify EOF is returned also after first time it is returned - _, err = j.Read(outBuffer) - if !errors.Is(err, io.EOF) { - t.Fatal("expected io.EOF") - } + // verify EOF is returned also after first time it is returned + _, err = j.Read(outBuffer) + if !errors.Is(err, io.EOF) { + t.Fatal("expected io.EOF") + } - _, err = j.Read(outBuffer) - if !errors.Is(err, io.EOF) { - t.Fatal("expected io.EOF") - } + _, err = j.Read(outBuffer) + if !errors.Is(err, io.EOF) { + t.Fatal("expected io.EOF") + } + }) } // TestJoinerTwoLevelsAcrossChunk tests the retrieval of data chunks below // first intermediate level across two intermediate chunks. // Last chunk has sub-chunk length. func TestJoinerTwoLevelsAcrossChunk(t *testing.T) { - t.Parallel() + synctest.Test(t, func(t *testing.T) { + store := inmemchunkstore.New() - store := inmemchunkstore.New() - - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() + ctx, cancel := context.WithTimeout(t.Context(), time.Second) + defer cancel() - // create root chunk with 2 references and two intermediate chunks with references - rootChunk := filetest.GenerateTestRandomFileChunk(swarm.ZeroAddress, swarm.ChunkSize*swarm.Branches+42, swarm.SectionSize*2) - err := store.Put(ctx, rootChunk) - if err != nil { - t.Fatal(err) - } + // create root chunk with 2 references and two intermediate chunks with references + rootChunk := filetest.GenerateTestRandomFileChunk(swarm.ZeroAddress, swarm.ChunkSize*swarm.Branches+42, swarm.SectionSize*2) + err := store.Put(ctx, rootChunk) + if err != nil { + t.Fatal(err) + } - firstAddress := swarm.NewAddress(rootChunk.Data()[8 : swarm.SectionSize+8]) - firstChunk := filetest.GenerateTestRandomFileChunk(firstAddress, swarm.ChunkSize*swarm.Branches, swarm.ChunkSize) - err = store.Put(ctx, firstChunk) - if err != nil { - t.Fatal(err) - } + firstAddress := swarm.NewAddress(rootChunk.Data()[8 : swarm.SectionSize+8]) + firstChunk := filetest.GenerateTestRandomFileChunk(firstAddress, swarm.ChunkSize*swarm.Branches, swarm.ChunkSize) + err = store.Put(ctx, firstChunk) + if err != nil { + t.Fatal(err) + } - secondAddress := swarm.NewAddress(rootChunk.Data()[swarm.SectionSize+8:]) - secondChunk := filetest.GenerateTestRandomFileChunk(secondAddress, 42, swarm.SectionSize) - err = store.Put(ctx, secondChunk) - if err != nil { - t.Fatal(err) - } + secondAddress := swarm.NewAddress(rootChunk.Data()[swarm.SectionSize+8:]) + secondChunk := filetest.GenerateTestRandomFileChunk(secondAddress, 42, swarm.SectionSize) + err = store.Put(ctx, secondChunk) + if err != nil { + t.Fatal(err) + } - // create 128+1 chunks for all references in the intermediate chunks - cursor := 8 - for range swarm.Branches { - chunkAddressBytes := firstChunk.Data()[cursor : cursor+swarm.SectionSize] + // create 128+1 chunks for all references in the intermediate chunks + cursor := 8 + for range swarm.Branches { + chunkAddressBytes := firstChunk.Data()[cursor : cursor+swarm.SectionSize] + chunkAddress := swarm.NewAddress(chunkAddressBytes) + ch := filetest.GenerateTestRandomFileChunk(chunkAddress, swarm.ChunkSize, swarm.ChunkSize) + err := store.Put(ctx, ch) + if err != nil { + t.Fatal(err) + } + cursor += swarm.SectionSize + } + chunkAddressBytes := secondChunk.Data()[8:] chunkAddress := swarm.NewAddress(chunkAddressBytes) - ch := filetest.GenerateTestRandomFileChunk(chunkAddress, swarm.ChunkSize, swarm.ChunkSize) - err := store.Put(ctx, ch) + ch := filetest.GenerateTestRandomFileChunk(chunkAddress, 42, 42) + err = store.Put(ctx, ch) if err != nil { t.Fatal(err) } - cursor += swarm.SectionSize - } - chunkAddressBytes := secondChunk.Data()[8:] - chunkAddress := swarm.NewAddress(chunkAddressBytes) - ch := filetest.GenerateTestRandomFileChunk(chunkAddress, 42, 42) - err = store.Put(ctx, ch) - if err != nil { - t.Fatal(err) - } - j, _, err := joiner.New(ctx, store, store, rootChunk.Address(), redundancy.DefaultLevel) - if err != nil { - t.Fatal(err) - } + j, _, err := joiner.New(ctx, store, store, rootChunk.Address(), redundancy.DefaultLevel) + if err != nil { + t.Fatal(err) + } - // read back all the chunks and verify - b := make([]byte, swarm.ChunkSize) - for i := range swarm.Branches { + // read back all the chunks and verify + b := make([]byte, swarm.ChunkSize) + for i := range swarm.Branches { + c, err := j.Read(b) + if err != nil { + t.Fatal(err) + } + if c != swarm.ChunkSize { + t.Fatalf("chunk %d expected read %d bytes; got %d", i, swarm.ChunkSize, c) + } + } c, err := j.Read(b) if err != nil { t.Fatal(err) } - if c != swarm.ChunkSize { - t.Fatalf("chunk %d expected read %d bytes; got %d", i, swarm.ChunkSize, c) + if c != 42 { + t.Fatalf("last chunk expected read %d bytes; got %d", 42, c) } - } - c, err := j.Read(b) - if err != nil { - t.Fatal(err) - } - if c != 42 { - t.Fatalf("last chunk expected read %d bytes; got %d", 42, c) - } + }) } func TestJoinerIterateChunkAddresses(t *testing.T) { - t.Parallel() - - store := inmemchunkstore.New() + synctest.Test(t, func(t *testing.T) { + store := inmemchunkstore.New() - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() + ctx, cancel := context.WithTimeout(t.Context(), time.Second) + defer cancel() - // create root chunk with 2 references and the referenced data chunks - rootChunk := filetest.GenerateTestRandomFileChunk(swarm.ZeroAddress, swarm.ChunkSize*2, swarm.SectionSize*2) - err := store.Put(ctx, rootChunk) - if err != nil { - t.Fatal(err) - } + // create root chunk with 2 references and the referenced data chunks + rootChunk := filetest.GenerateTestRandomFileChunk(swarm.ZeroAddress, swarm.ChunkSize*2, swarm.SectionSize*2) + err := store.Put(ctx, rootChunk) + if err != nil { + t.Fatal(err) + } - firstAddress := swarm.NewAddress(rootChunk.Data()[8 : swarm.SectionSize+8]) - firstChunk := filetest.GenerateTestRandomFileChunk(firstAddress, swarm.ChunkSize, swarm.ChunkSize) - err = store.Put(ctx, firstChunk) - if err != nil { - t.Fatal(err) - } + firstAddress := swarm.NewAddress(rootChunk.Data()[8 : swarm.SectionSize+8]) + firstChunk := filetest.GenerateTestRandomFileChunk(firstAddress, swarm.ChunkSize, swarm.ChunkSize) + err = store.Put(ctx, firstChunk) + if err != nil { + t.Fatal(err) + } - secondAddress := swarm.NewAddress(rootChunk.Data()[swarm.SectionSize+8:]) - secondChunk := filetest.GenerateTestRandomFileChunk(secondAddress, swarm.ChunkSize, swarm.ChunkSize) - err = store.Put(ctx, secondChunk) - if err != nil { - t.Fatal(err) - } + secondAddress := swarm.NewAddress(rootChunk.Data()[swarm.SectionSize+8:]) + secondChunk := filetest.GenerateTestRandomFileChunk(secondAddress, swarm.ChunkSize, swarm.ChunkSize) + err = store.Put(ctx, secondChunk) + if err != nil { + t.Fatal(err) + } - createdAddresses := []swarm.Address{rootChunk.Address(), firstAddress, secondAddress} + createdAddresses := []swarm.Address{rootChunk.Address(), firstAddress, secondAddress} - j, _, err := joiner.New(ctx, store, store, rootChunk.Address(), redundancy.DefaultLevel) - if err != nil { - t.Fatal(err) - } + j, _, err := joiner.New(ctx, store, store, rootChunk.Address(), redundancy.DefaultLevel) + if err != nil { + t.Fatal(err) + } - foundAddresses := make(map[string]struct{}) - var foundAddressesMu sync.Mutex + foundAddresses := make(map[string]struct{}) + var foundAddressesMu sync.Mutex - err = j.IterateChunkAddresses(func(addr swarm.Address) error { - foundAddressesMu.Lock() - defer foundAddressesMu.Unlock() + err = j.IterateChunkAddresses(func(addr swarm.Address) error { + foundAddressesMu.Lock() + defer foundAddressesMu.Unlock() - foundAddresses[addr.String()] = struct{}{} - return nil - }) - if err != nil { - t.Fatal(err) - } + foundAddresses[addr.String()] = struct{}{} + return nil + }) + if err != nil { + t.Fatal(err) + } - if len(createdAddresses) != len(foundAddresses) { - t.Fatalf("expected to find %d addresses, got %d", len(createdAddresses), len(foundAddresses)) - } + if len(createdAddresses) != len(foundAddresses) { + t.Fatalf("expected to find %d addresses, got %d", len(createdAddresses), len(foundAddresses)) + } - checkAddressFound := func(t *testing.T, foundAddresses map[string]struct{}, address swarm.Address) { - t.Helper() + checkAddressFound := func(t *testing.T, foundAddresses map[string]struct{}, address swarm.Address) { + t.Helper() - if _, ok := foundAddresses[address.String()]; !ok { - t.Fatalf("expected address %s not found", address.String()) + if _, ok := foundAddresses[address.String()]; !ok { + t.Fatalf("expected address %s not found", address.String()) + } } - } - for _, createdAddress := range createdAddresses { - checkAddressFound(t, foundAddresses, createdAddress) - } + for _, createdAddress := range createdAddresses { + checkAddressFound(t, foundAddresses, createdAddress) + } + }) } func TestJoinerIterateChunkAddresses_Encrypted(t *testing.T) { @@ -1109,7 +1111,7 @@ func TestJoinerRedundancy(t *testing.T) { } // all data can be read back readCheck := func(t *testing.T, expErr error) { - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(t.Context()) defer cancel() decodeTimeoutStr := time.Second.String() @@ -1226,146 +1228,150 @@ func TestJoinerRedundancy(t *testing.T) { // // nolint:thelper func TestJoinerRedundancyMultilevel(t *testing.T) { - t.Parallel() - test := func(t *testing.T, rLevel redundancy.Level, encrypt bool, size int) { - t.Helper() - store := mockstorer.NewForgettingStore(newChunkStore()) - seed, err := pseudorand.NewSeed() - if err != nil { - t.Fatal(err) - } - dataReader := pseudorand.NewReader(seed, size*swarm.ChunkSize) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - pipe := builder.NewPipelineBuilder(ctx, store, encrypt, rLevel) - addr, err := builder.FeedPipeline(ctx, pipe, dataReader) - if err != nil { - t.Fatal(err) - } - expRead := swarm.ChunkSize - buf := make([]byte, expRead) - offset := mrand.Intn(size) * expRead - canReadRange := func(t *testing.T, s getter.Strategy, fallback bool, canRead bool) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + r2level := []int{2, 1, 2, 3, 2} + encryptChunk := []bool{false, false, true, true, true} - decodingTimeoutStr := time.Second.String() + for _, rLevel := range []redundancy.Level{0, 1, 2, 3, 4} { + for _, encrypt := range []bool{false, true} { + for _, levels := range []int{1, 2, 3} { + // Skip tests that don't meet the filter criteria to save time + if r2level[rLevel] != levels || encrypt != encryptChunk[rLevel] { + t.Skip("skipping to save time") + continue + } - ctx, err := getter.SetConfigInContext(ctx, &s, &fallback, &decodingTimeoutStr, log.Noop) - if err != nil { - t.Fatal(err) - } + // Calculate chunk count + shardCnt := rLevel.GetMaxShards() + if encrypt { + shardCnt = rLevel.GetMaxEncShards() + } - j, _, err := joiner.New(ctx, store, store, addr, redundancy.DefaultLevel) - if err != nil { - t.Fatal(err) - } - n, err := j.ReadAt(buf, int64(offset)) - if !canRead { - if !errors.Is(err, storage.ErrNotFound) && !errors.Is(err, context.DeadlineExceeded) { - t.Fatalf("expected error %v or %v. got %v", storage.ErrNotFound, context.DeadlineExceeded, err) + var chunkCnt int + switch levels { + case 1: + chunkCnt = 2 + case 2: + chunkCnt = shardCnt + 1 + case 3: + chunkCnt = shardCnt*shardCnt + 1 } - return - } - if err != nil { - t.Fatal(err) - } - if n != expRead { - t.Errorf("read %d bytes out of %d", n, expRead) - } - _, err = dataReader.Seek(int64(offset), io.SeekStart) - if err != nil { - t.Fatal(err) - } - ok, err := dataReader.Match(bytes.NewBuffer(buf), expRead) - if err != nil { - t.Fatal(err) - } - if !ok { - t.Error("content mismatch") + + testName := fmt.Sprintf("rLevel=%v_encrypt=%v_levels=%d_chunks=%d_incomplete", rLevel, encrypt, levels, chunkCnt) + t.Run(testName, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + runRedundancyTest(t, rLevel, encrypt, chunkCnt) + }) + }) + + // For the "full" test cases + switch levels { + case 1: + chunkCnt = shardCnt + case 2: + chunkCnt = shardCnt * shardCnt + case 3: + continue + } + + testName = fmt.Sprintf("rLevel=%v_encrypt=%v_levels=%d_chunks=%d_full", rLevel, encrypt, levels, chunkCnt) + t.Run(testName, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + runRedundancyTest(t, rLevel, encrypt, chunkCnt) + }) + }) } } + } +} - // first sanity check and recover a range - t.Run("NONE w/o fallback CAN retrieve", func(t *testing.T) { - store.Record() - defer store.Unrecord() - canReadRange(t, getter.NONE, false, true) - }) +// Helper function to run the redundancy test without nested t.Run calls +func runRedundancyTest(t *testing.T, rLevel redundancy.Level, encrypt bool, size int) { + t.Helper() - // do not forget the root chunk - store.Unmiss(swarm.NewAddress(addr.Bytes()[:swarm.HashSize])) - // after we forget the chunks on the way to the range, we should not be able to retrieve - t.Run("NONE w/o fallback CANNOT retrieve", func(t *testing.T) { - canReadRange(t, getter.NONE, false, false) - }) + store := mockstorer.NewForgettingStore(newChunkStore()) + seed, err := pseudorand.NewSeed() + if err != nil { + t.Fatal(err) + } + dataReader := pseudorand.NewReader(seed, size*swarm.ChunkSize) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + pipe := builder.NewPipelineBuilder(ctx, store, encrypt, rLevel) + addr, err := builder.FeedPipeline(ctx, pipe, dataReader) + if err != nil { + t.Fatal(err) + } + expRead := swarm.ChunkSize + buf := make([]byte, expRead) + offset := mrand.Intn(size) * expRead - // we lost a data chunk, we cannot recover using DATA only strategy with no fallback - t.Run("DATA w/o fallback CANNOT retrieve", func(t *testing.T) { - canReadRange(t, getter.DATA, false, false) - }) + // Helper function to test reading with different strategies + canReadRange := func(s getter.Strategy, fallback bool, canRead bool, description string) { + t.Logf("Testing: %s", description) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() - if rLevel == 0 { - // allowing fallback mode will not help if no redundancy used for upload - t.Run("DATA w fallback CANNOT retrieve", func(t *testing.T) { - canReadRange(t, getter.DATA, true, false) - }) + decodingTimeoutStr := time.Second.String() + + ctx, err := getter.SetConfigInContext(ctx, &s, &fallback, &decodingTimeoutStr, log.Noop) + if err != nil { + t.Fatal(err) + } + + j, _, err := joiner.New(ctx, store, store, addr, redundancy.DefaultLevel) + if err != nil { + t.Fatal(err) + } + n, err := j.ReadAt(buf, int64(offset)) + if !canRead { + if !errors.Is(err, storage.ErrNotFound) && !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("expected error %v or %v. got %v", storage.ErrNotFound, context.DeadlineExceeded, err) + } return } - // allowing fallback mode will make the range retrievable using erasure decoding - t.Run("DATA w fallback CAN retrieve", func(t *testing.T) { - canReadRange(t, getter.DATA, true, true) - }) - // after the reconstructed data is stored, we can retrieve the range using DATA only mode - t.Run("after recovery, NONE w/o fallback CAN retrieve", func(t *testing.T) { - canReadRange(t, getter.NONE, false, true) - }) + if err != nil { + t.Fatal(err) + } + if n != expRead { + t.Errorf("read %d bytes out of %d", n, expRead) + } + if _, err = dataReader.Seek(int64(offset), io.SeekStart); err != nil { + t.Fatal(err) + } + ok, err := dataReader.Match(bytes.NewBuffer(buf), expRead) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Error("content mismatch") + } } - r2level := []int{2, 1, 2, 3, 2} - encryptChunk := []bool{false, false, true, true, true} - for _, rLevel := range []redundancy.Level{0, 1, 2, 3, 4} { - // speeding up tests by skipping some of them - t.Run(fmt.Sprintf("rLevel=%v", rLevel), func(t *testing.T) { - t.Parallel() - for _, encrypt := range []bool{false, true} { - shardCnt := rLevel.GetMaxShards() - if encrypt { - shardCnt = rLevel.GetMaxEncShards() - } - for _, levels := range []int{1, 2, 3} { - chunkCnt := 1 - switch levels { - case 1: - chunkCnt = 2 - case 2: - chunkCnt = shardCnt + 1 - case 3: - chunkCnt = shardCnt*shardCnt + 1 - } - t.Run(fmt.Sprintf("encrypt=%v levels=%d chunks=%d incomplete", encrypt, levels, chunkCnt), func(t *testing.T) { - if r2level[rLevel] != levels || encrypt != encryptChunk[rLevel] { - t.Skip("skipping to save time") - } - test(t, rLevel, encrypt, chunkCnt) - }) - switch levels { - case 1: - chunkCnt = shardCnt - case 2: - chunkCnt = shardCnt * shardCnt - case 3: - continue - } - t.Run(fmt.Sprintf("encrypt=%v levels=%d chunks=%d full", encrypt, levels, chunkCnt), func(t *testing.T) { - if r2level[rLevel] != levels || encrypt != encryptChunk[rLevel] { - t.Skip("skipping to save time") - } - test(t, rLevel, encrypt, chunkCnt) - }) - } - } - }) + + // First sanity check and recover a range + store.Record() + canReadRange(getter.NONE, false, true, "NONE w/o fallback CAN retrieve") + store.Unrecord() + + // Do not forget the root chunk + store.Unmiss(swarm.NewAddress(addr.Bytes()[:swarm.HashSize])) + + // After we forget the chunks on the way to the range, we should not be able to retrieve + canReadRange(getter.NONE, false, false, "NONE w/o fallback CANNOT retrieve") + + // We lost a data chunk, we cannot recover using DATA only strategy with no fallback + canReadRange(getter.DATA, false, false, "DATA w/o fallback CANNOT retrieve") + + if rLevel == 0 { + // Allowing fallback mode will not help if no redundancy used for upload + canReadRange(getter.DATA, true, false, "DATA w fallback CANNOT retrieve") + return } + + // Allowing fallback mode will make the range retrievable using erasure decoding + canReadRange(getter.DATA, true, true, "DATA w fallback CAN retrieve") + + // After the reconstructed data is stored, we can retrieve the range using DATA only mode + canReadRange(getter.NONE, false, true, "after recovery, NONE w/o fallback CAN retrieve") } type chunkStore struct { From 26530f12fd16c01b0d7adb823efb9534ee9d83a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ljubi=C5=A1a=20Ga=C4=8Devi=C4=87?= <35105035+gacevicljubisa@users.noreply.github.com> Date: Wed, 21 Jan 2026 12:22:08 +0100 Subject: [PATCH 72/96] chore: update postage snapshot (#5325) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 1cee528864c..cf602914109 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/caddyserver/certmagic v0.21.6 github.com/coreos/go-semver v0.3.0 github.com/ethereum/go-ethereum v1.15.11 - github.com/ethersphere/batch-archive v0.0.4 + github.com/ethersphere/batch-archive v0.0.5 github.com/ethersphere/go-price-oracle-abi v0.6.9 github.com/ethersphere/go-storage-incentives-abi v0.9.4 github.com/ethersphere/go-sw3-abi v0.6.9 diff --git a/go.sum b/go.sum index 7b0f87f302a..fab9e1c2f69 100644 --- a/go.sum +++ b/go.sum @@ -236,8 +236,8 @@ github.com/ethereum/go-ethereum v1.15.11 h1:JK73WKeu0WC0O1eyX+mdQAVHUV+UR1a9VB/d github.com/ethereum/go-ethereum v1.15.11/go.mod h1:mf8YiHIb0GR4x4TipcvBUPxJLw1mFdmxzoDi11sDRoI= github.com/ethereum/go-verkle v0.2.2 h1:I2W0WjnrFUIzzVPwm8ykY+7pL2d4VhlsePn4j7cnFk8= github.com/ethereum/go-verkle v0.2.2/go.mod h1:M3b90YRnzqKyyzBEWJGqj8Qff4IDeXnzFw0P9bFw3uk= -github.com/ethersphere/batch-archive v0.0.4 h1:PHmwQfmUEyDJgoX2IqI/R0alQ63+aLPXfdNNSmNLGwI= -github.com/ethersphere/batch-archive v0.0.4/go.mod h1:41BPb192NoK9CYjNB8BAE1J2MtiI/5aq0Wtas5O7A7Q= +github.com/ethersphere/batch-archive v0.0.5 h1:SM3g7Tuge4KhOn+NKgPcg2Uz2p8a/MLKZvZpmkKCyU4= +github.com/ethersphere/batch-archive v0.0.5/go.mod h1:41BPb192NoK9CYjNB8BAE1J2MtiI/5aq0Wtas5O7A7Q= github.com/ethersphere/go-price-oracle-abi v0.6.9 h1:bseen6he3PZv5GHOm+KD6s4awaFmVSD9LFx+HpB6rCU= github.com/ethersphere/go-price-oracle-abi v0.6.9/go.mod h1:sI/Qj4/zJ23/b1enzwMMv0/hLTpPNVNacEwCWjo6yBk= github.com/ethersphere/go-storage-incentives-abi v0.9.4 h1:mSIWXQXg5OQmH10QvXMV5w0vbSibFMaRlBL37gPLTM0= From eccd4f13a45a3165ef2955f73c5c941792fb1f24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ljubi=C5=A1a=20Ga=C4=8Devi=C4=87?= <35105035+gacevicljubisa@users.noreply.github.com> Date: Mon, 26 Jan 2026 16:58:28 +0100 Subject: [PATCH 73/96] fix(p2p): handle pre-release versions in bee260 backward compatibility check (#5327) --- pkg/p2p/libp2p/libp2p.go | 18 +++- pkg/p2p/libp2p/version_test.go | 165 +++++++++++++++++++++++++++++++++ 2 files changed, 180 insertions(+), 3 deletions(-) create mode 100644 pkg/p2p/libp2p/version_test.go diff --git a/pkg/p2p/libp2p/libp2p.go b/pkg/p2p/libp2p/libp2p.go index 6bab9b8214c..206615840c3 100644 --- a/pkg/p2p/libp2p/libp2p.go +++ b/pkg/p2p/libp2p/libp2p.go @@ -655,11 +655,13 @@ func (s *Service) handleIncoming(stream network.Stream) { return } + bee260Compat := s.bee260BackwardCompatibility(peerID) + i, err := s.handshakeService.Handle( s.ctx, handshakeStream, peerMultiaddrs, - handshake.WithBee260Compatibility(s.bee260BackwardCompatibility(peerID)), + handshake.WithBee260Compatibility(bee260Compat), ) if err != nil { s.logger.Debug("stream handler: handshake: handle failed", "peer_id", peerID, "error", err) @@ -1070,11 +1072,13 @@ func (s *Service) Connect(ctx context.Context, addrs []ma.Multiaddr) (address *b return nil, fmt.Errorf("build peer multiaddrs: %w", err) } + bee260Compat := s.bee260BackwardCompatibility(peerID) + i, err := s.handshakeService.Handshake( s.ctx, handshakeStream, peerMultiaddrs, - handshake.WithBee260Compatibility(s.bee260BackwardCompatibility(peerID)), + handshake.WithBee260Compatibility(bee260Compat), ) if err != nil { _ = handshakeStream.Reset() @@ -1484,7 +1488,15 @@ func (s *Service) bee260BackwardCompatibility(peerID libp2ppeer.ID) bool { if err != nil { return false } - return v.LessThan(version270) + + // Compare major.minor.patch only (ignore pre-release) + // This way 2.7.0-rc12 is treated as >= 2.7.0 + vCore, err := semver.NewVersion(fmt.Sprintf("%d.%d.%d", v.Major, v.Minor, v.Patch)) + if err != nil { + return false + } + result := vCore.LessThan(version270) + return result } // appendSpace adds a leading space character if the string is not empty. diff --git a/pkg/p2p/libp2p/version_test.go b/pkg/p2p/libp2p/version_test.go new file mode 100644 index 00000000000..021b61b818a --- /dev/null +++ b/pkg/p2p/libp2p/version_test.go @@ -0,0 +1,165 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package libp2p + +import ( + "context" + "testing" + + "github.com/ethersphere/bee/v2/pkg/crypto" + "github.com/ethersphere/bee/v2/pkg/log" + "github.com/ethersphere/bee/v2/pkg/statestore/mock" + "github.com/ethersphere/bee/v2/pkg/swarm" + libp2ppeer "github.com/libp2p/go-libp2p/core/peer" +) + +func TestBee260BackwardCompatibility(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + userAgent string + want bool + }{ + // Versions < 2.7.0 should require backward compatibility + { + name: "version 2.6.0", + userAgent: "bee/2.6.0 go1.22.0 linux/amd64", + want: true, + }, + { + name: "version 2.6.5", + userAgent: "bee/2.6.5 go1.22.0 linux/amd64", + want: true, + }, + { + name: "version 2.5.0", + userAgent: "bee/2.5.0 go1.21.0 linux/amd64", + want: true, + }, + { + name: "version 2.6.0-beta1", + userAgent: "bee/2.6.0-beta1 go1.22.0 linux/amd64", + want: true, + }, + // Versions >= 2.7.0 should NOT require backward compatibility + { + name: "version 2.7.0", + userAgent: "bee/2.7.0 go1.23.0 linux/amd64", + want: false, + }, + { + name: "version 2.8.0", + userAgent: "bee/2.8.0 go1.23.0 linux/amd64", + want: false, + }, + { + name: "version 3.0.0", + userAgent: "bee/3.0.0 go1.25.0 linux/amd64", + want: false, + }, + // Pre-release versions >= 2.7.0 should NOT require backward compatibility + // This is the critical fix: 2.7.0-rcX should be treated as >= 2.7.0 + { + name: "version 2.7.0-rc1", + userAgent: "bee/2.7.0-rc1 go1.23.0 linux/amd64", + want: false, + }, + { + name: "version 2.7.0-rc12", + userAgent: "bee/2.7.0-rc12-b39629d5-dirty go1.25.6 linux/amd64", + want: false, + }, + { + name: "version 2.7.0-beta1", + userAgent: "bee/2.7.0-beta1 go1.23.0 linux/amd64", + want: false, + }, + { + name: "version 2.8.0-rc1", + userAgent: "bee/2.8.0-rc1 go1.24.0 linux/amd64", + want: false, + }, + { + name: "version 2.9.0-beta1", + userAgent: "bee/2.9.0-beta1 go1.24.0 linux/amd64", + want: false, + }, + // Edge cases that should return false (not requiring backward compat) + { + name: "empty user agent", + userAgent: "", + want: false, + }, + { + name: "malformed user agent missing space", + userAgent: "bee/2.6.0", + want: false, + }, + { + name: "non-bee user agent", + userAgent: "other/1.0.0 go1.22.0 linux/amd64", + want: false, + }, + { + name: "invalid version format", + userAgent: "bee/invalid go1.22.0 linux/amd64", + want: false, + }, + { + name: "default libp2p user agent", + userAgent: "github.com/libp2p/go-libp2p", + want: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + // Create a service with minimal configuration + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + swarmKey, err := crypto.GenerateSecp256k1Key() + if err != nil { + t.Fatal(err) + } + + overlay := swarm.RandAddress(t) + addr := ":0" + networkID := uint64(1) + + statestore := mock.NewStateStore() + defer statestore.Close() + + s, err := New(ctx, crypto.NewDefaultSigner(swarmKey), networkID, overlay, addr, nil, statestore, nil, log.Noop, nil, Options{}) + if err != nil { + t.Fatal(err) + } + defer s.Close() + + // Create a random test peer ID - we only need any valid libp2p peer ID + // The peerstore lookup will be mocked by setting the AgentVersion directly + libp2pPeerID, err := libp2ppeer.Decode("16Uiu2HAm3g4hXfCWTDhPBq3KkqpV3wGkPVgMJY3Jt8gGTYWiTWNZ") + if err != nil { + t.Fatal(err) + } + + // Set the user agent in the peerstore if provided + if tc.userAgent != "" { + if err := s.host.Peerstore().Put(libp2pPeerID, "AgentVersion", tc.userAgent); err != nil { + t.Fatal(err) + } + } + + // Test the backward compatibility check + got := s.bee260BackwardCompatibility(libp2pPeerID) + if got != tc.want { + t.Errorf("bee260BackwardCompatibility() = %v, want %v (userAgent: %q)", got, tc.want, tc.userAgent) + } + }) + } +} From 5ad0417acee227de12ed8744944680d9a243ff2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jano=C5=A1=20Gulja=C5=A1?= Date: Tue, 27 Jan 2026 12:58:01 +0100 Subject: [PATCH 74/96] fix: make hive bee 2.6.0 backward compatible (#5329) Co-authored-by: Ljubisa Gacevic --- pkg/hive/hive.go | 6 +- .../libp2p/internal/handshake/handshake.go | 23 ++----- pkg/p2p/libp2p/libp2p.go | 15 ++++ pkg/p2p/libp2p/peer.go | 39 ++++++++--- pkg/p2p/libp2p/version_test.go | 69 +++++++++++++++++++ pkg/p2p/p2p.go | 21 ++++++ pkg/p2p/streamtest/streamtest.go | 6 ++ 7 files changed, 147 insertions(+), 32 deletions(-) diff --git a/pkg/hive/hive.go b/pkg/hive/hive.go index 1f49fe2808c..8f8572bcb34 100644 --- a/pkg/hive/hive.go +++ b/pkg/hive/hive.go @@ -50,7 +50,7 @@ var ( ) type Service struct { - streamer p2p.Streamer + streamer p2p.Bee260CompatibilityStreamer addressBook addressbook.GetPutter addPeersHandler func(...swarm.Address) networkID uint64 @@ -67,7 +67,7 @@ type Service struct { overlay swarm.Address } -func New(streamer p2p.Streamer, addressbook addressbook.GetPutter, networkID uint64, bootnode bool, allowPrivateCIDRs bool, overlay swarm.Address, logger log.Logger) *Service { +func New(streamer p2p.Bee260CompatibilityStreamer, addressbook addressbook.GetPutter, networkID uint64, bootnode bool, allowPrivateCIDRs bool, overlay swarm.Address, logger log.Logger) *Service { svc := &Service{ streamer: streamer, logger: logger.WithName(loggerName).Register(), @@ -196,6 +196,8 @@ func (s *Service) sendPeers(ctx context.Context, peer swarm.Address, peers []swa continue } + advertisableUnderlays = p2p.FilterBee260CompatibleUnderlays(s.streamer.IsBee260(peer), advertisableUnderlays) + peersRequest.Peers = append(peersRequest.Peers, &pb.BzzAddress{ Overlay: addr.Overlay.Bytes(), Underlay: bzz.SerializeUnderlays(advertisableUnderlays), diff --git a/pkg/p2p/libp2p/internal/handshake/handshake.go b/pkg/p2p/libp2p/internal/handshake/handshake.go index 0dcc726d7d7..63977652601 100644 --- a/pkg/p2p/libp2p/internal/handshake/handshake.go +++ b/pkg/p2p/libp2p/internal/handshake/handshake.go @@ -153,7 +153,7 @@ func (s *Service) Handshake(ctx context.Context, stream p2p.Stream, peerMultiadd w, r := protobuf.NewWriterAndReader(stream) - peerMultiaddrs = filterBee260CompatibleUnderlays(o.bee260compatibility, peerMultiaddrs) + peerMultiaddrs = p2p.FilterBee260CompatibleUnderlays(o.bee260compatibility, peerMultiaddrs) if err := w.WriteMsgWithContext(ctx, &pb.Syn{ ObservedUnderlay: bzz.SerializeUnderlays(peerMultiaddrs), @@ -208,7 +208,7 @@ func (s *Service) Handshake(ctx context.Context, stream p2p.Stream, peerMultiadd return a.Equal(b) }) - advertisableUnderlays = filterBee260CompatibleUnderlays(o.bee260compatibility, advertisableUnderlays) + advertisableUnderlays = p2p.FilterBee260CompatibleUnderlays(o.bee260compatibility, advertisableUnderlays) bzzAddress, err := bzz.NewAddress(s.signer, advertisableUnderlays, s.overlay, s.networkID, s.nonce) if err != nil { @@ -306,7 +306,7 @@ func (s *Service) Handle(ctx context.Context, stream p2p.Stream, peerMultiaddrs return a.Equal(b) }) - advertisableUnderlays = filterBee260CompatibleUnderlays(o.bee260compatibility, advertisableUnderlays) + advertisableUnderlays = p2p.FilterBee260CompatibleUnderlays(o.bee260compatibility, advertisableUnderlays) bzzAddress, err := bzz.NewAddress(s.signer, advertisableUnderlays, s.overlay, s.networkID, s.nonce) if err != nil { @@ -315,7 +315,7 @@ func (s *Service) Handle(ctx context.Context, stream p2p.Stream, peerMultiaddrs welcomeMessage := s.GetWelcomeMessage() - peerMultiaddrs = filterBee260CompatibleUnderlays(o.bee260compatibility, peerMultiaddrs) + peerMultiaddrs = p2p.FilterBee260CompatibleUnderlays(o.bee260compatibility, peerMultiaddrs) if err := w.WriteMsgWithContext(ctx, &pb.SynAck{ Syn: &pb.Syn{ @@ -395,18 +395,3 @@ func (s *Service) parseCheckAck(ack *pb.Ack) (*bzz.Address, error) { return bzzAddress, nil } - -// filterBee260CompatibleUnderlays select a single underlay to pass if -// bee260compatibility is true. Otherwise it passes the unmodified underlays -// slice. This function can be safely removed when bee version 2.6.0 is -// deprecated. -func filterBee260CompatibleUnderlays(bee260compatibility bool, underlays []ma.Multiaddr) []ma.Multiaddr { - if !bee260compatibility { - return underlays - } - underlay := bzz.SelectBestAdvertisedAddress(underlays, nil) - if underlay == nil { - return underlays - } - return []ma.Multiaddr{underlay} -} diff --git a/pkg/p2p/libp2p/libp2p.go b/pkg/p2p/libp2p/libp2p.go index 206615840c3..e4edacb3453 100644 --- a/pkg/p2p/libp2p/libp2p.go +++ b/pkg/p2p/libp2p/libp2p.go @@ -1475,9 +1475,23 @@ func (s *Service) peerMultiaddrs(ctx context.Context, peerID libp2ppeer.ID) ([]m return buildFullMAs(waitPeerAddrs(waitPeersCtx, s.host.Peerstore(), peerID), peerID) } +// IsBee260 implements p2p.Bee260CompatibilityStreamer interface. +// It checks if a peer is running Bee version older than 2.7.0. +func (s *Service) IsBee260(overlay swarm.Address) bool { + peerID, found := s.peers.peerID(overlay) + if !found { + return false + } + return s.bee260BackwardCompatibility(peerID) +} + var version270 = *semver.Must(semver.NewVersion("2.7.0")) func (s *Service) bee260BackwardCompatibility(peerID libp2ppeer.ID) bool { + if compat, found := s.peers.bee260(peerID); found { + return compat + } + userAgent := s.peerUserAgent(s.ctx, peerID) p := strings.SplitN(userAgent, " ", 2) if len(p) != 2 { @@ -1496,6 +1510,7 @@ func (s *Service) bee260BackwardCompatibility(peerID libp2ppeer.ID) bool { return false } result := vCore.LessThan(version270) + s.peers.setBee260(peerID, result) return result } diff --git a/pkg/p2p/libp2p/peer.go b/pkg/p2p/libp2p/peer.go index 7966454f295..5fcb32303f0 100644 --- a/pkg/p2p/libp2p/peer.go +++ b/pkg/p2p/libp2p/peer.go @@ -18,12 +18,13 @@ import ( ) type peerRegistry struct { - underlays map[string]libp2ppeer.ID // map overlay address to underlay peer id - overlays map[libp2ppeer.ID]swarm.Address // map underlay peer id to overlay address - full map[libp2ppeer.ID]bool // map to track whether a node is full or light node (true=full) - connections map[libp2ppeer.ID]map[network.Conn]struct{} // list of connections for safe removal on Disconnect notification - streams map[libp2ppeer.ID]map[network.Stream]context.CancelFunc - mu sync.RWMutex + underlays map[string]libp2ppeer.ID // map overlay address to underlay peer id + overlays map[libp2ppeer.ID]swarm.Address // map underlay peer id to overlay address + full map[libp2ppeer.ID]bool // map to track whether a node is full or light node (true=full) + bee260Compatibility map[libp2ppeer.ID]bool // map to track bee260 backward compatibility + connections map[libp2ppeer.ID]map[network.Conn]struct{} // list of connections for safe removal on Disconnect notification + streams map[libp2ppeer.ID]map[network.Stream]context.CancelFunc + mu sync.RWMutex //nolint:misspell disconnecter disconnecter // peerRegistry notifies libp2p on peer disconnection @@ -36,11 +37,12 @@ type disconnecter interface { func newPeerRegistry() *peerRegistry { return &peerRegistry{ - underlays: make(map[string]libp2ppeer.ID), - overlays: make(map[libp2ppeer.ID]swarm.Address), - full: make(map[libp2ppeer.ID]bool), - connections: make(map[libp2ppeer.ID]map[network.Conn]struct{}), - streams: make(map[libp2ppeer.ID]map[network.Stream]context.CancelFunc), + underlays: make(map[string]libp2ppeer.ID), + overlays: make(map[libp2ppeer.ID]swarm.Address), + full: make(map[libp2ppeer.ID]bool), + bee260Compatibility: make(map[libp2ppeer.ID]bool), + connections: make(map[libp2ppeer.ID]map[network.Conn]struct{}), + streams: make(map[libp2ppeer.ID]map[network.Stream]context.CancelFunc), Notifiee: new(network.NoopNotifiee), } @@ -81,6 +83,7 @@ func (r *peerRegistry) Disconnected(_ network.Network, c network.Conn) { } delete(r.streams, peerID) delete(r.full, peerID) + delete(r.bee260Compatibility, peerID) r.mu.Unlock() r.disconnecter.disconnected(overlay) @@ -176,6 +179,19 @@ func (r *peerRegistry) fullnode(peerID libp2ppeer.ID) (bool, bool) { return full, found } +func (r *peerRegistry) bee260(peerID libp2ppeer.ID) (compat, found bool) { + r.mu.RLock() + defer r.mu.RUnlock() + compat, found = r.bee260Compatibility[peerID] + return compat, found +} + +func (r *peerRegistry) setBee260(peerID libp2ppeer.ID, compat bool) { + r.mu.Lock() + defer r.mu.Unlock() + r.bee260Compatibility[peerID] = compat +} + func (r *peerRegistry) isConnected(peerID libp2ppeer.ID, remoteAddr ma.Multiaddr) (swarm.Address, bool) { if remoteAddr == nil { return swarm.ZeroAddress, false @@ -217,6 +233,7 @@ func (r *peerRegistry) remove(overlay swarm.Address) (found, full bool, peerID l delete(r.streams, peerID) full = r.full[peerID] delete(r.full, peerID) + delete(r.bee260Compatibility, peerID) r.mu.Unlock() return found, full, peerID diff --git a/pkg/p2p/libp2p/version_test.go b/pkg/p2p/libp2p/version_test.go index 021b61b818a..15e0ce3eba6 100644 --- a/pkg/p2p/libp2p/version_test.go +++ b/pkg/p2p/libp2p/version_test.go @@ -163,3 +163,72 @@ func TestBee260BackwardCompatibility(t *testing.T) { }) } } + +func TestBee260Cache(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + swarmKey, err := crypto.GenerateSecp256k1Key() + if err != nil { + t.Fatal(err) + } + + overlay := swarm.RandAddress(t) + addr := ":0" + networkID := uint64(1) + + statestore := mock.NewStateStore() + defer statestore.Close() + + s, err := New(ctx, crypto.NewDefaultSigner(swarmKey), networkID, overlay, addr, nil, statestore, nil, log.Noop, nil, Options{}) + if err != nil { + t.Fatal(err) + } + defer s.Close() + + libp2pPeerID, err := libp2ppeer.Decode("16Uiu2HAm3g4hXfCWTDhPBq3KkqpV3wGkPVgMJY3Jt8gGTYWiTWNZ") + if err != nil { + t.Fatal(err) + } + + // 1. Set user agent to 2.6.0 (compat = true) + if err := s.host.Peerstore().Put(libp2pPeerID, "AgentVersion", "bee/2.6.0 go1.22.0 linux/amd64"); err != nil { + t.Fatal(err) + } + + // 2. First call should calculate and cache it + if !s.bee260BackwardCompatibility(libp2pPeerID) { + t.Fatal("expected true for 2.6.0") + } + + // 3. Verify it's in the cache + compat, found := s.peers.bee260(libp2pPeerID) + if !found { + t.Fatal("expected value to be in cache") + } + if !compat { + t.Fatal("expected cached value to be true") + } + + // 4. Change user agent in peerstore to 2.7.0 (compat = false) + // If caching works, bee260BackwardCompatibility should still return true + if err := s.host.Peerstore().Put(libp2pPeerID, "AgentVersion", "bee/2.7.0 go1.23.0 linux/amd64"); err != nil { + t.Fatal(err) + } + + if !s.bee260BackwardCompatibility(libp2pPeerID) { + t.Fatal("expected true (cached value) even if peerstore changed") + } + + // 5. Clear cache (manually for testing) + s.peers.mu.Lock() + delete(s.peers.bee260Compatibility, libp2pPeerID) + s.peers.mu.Unlock() + + // 6. Now it should re-calculate and return false for 2.7.0 + if s.bee260BackwardCompatibility(libp2pPeerID) { + t.Fatal("expected false for 2.7.0 after cache clear") + } +} diff --git a/pkg/p2p/p2p.go b/pkg/p2p/p2p.go index b709ffa5113..ca725f73dd2 100644 --- a/pkg/p2p/p2p.go +++ b/pkg/p2p/p2p.go @@ -141,6 +141,12 @@ type Streamer interface { NewStream(ctx context.Context, address swarm.Address, h Headers, protocol, version, stream string) (Stream, error) } +// Bee260CompatibilityStreamer is able to create a new Stream and check if a peer is running Bee 2.6.0. +type Bee260CompatibilityStreamer interface { + NewStream(ctx context.Context, address swarm.Address, h Headers, protocol, version, stream string) (Stream, error) + IsBee260(address swarm.Address) bool +} + type StreamerDisconnecter interface { Streamer Disconnecter @@ -237,3 +243,18 @@ func (e *ChunkDeliveryError) Error() string { func NewChunkDeliveryError(msg string) error { return &ChunkDeliveryError{msg: msg} } + +// FilterBee260CompatibleUnderlays select a single underlay to pass if +// bee260compatibility is true. Otherwise it passes the unmodified underlays +// slice. This function can be safely removed when bee version 2.6.0 is +// deprecated. +func FilterBee260CompatibleUnderlays(bee260compatibility bool, underlays []ma.Multiaddr) []ma.Multiaddr { + if !bee260compatibility { + return underlays + } + underlay := bzz.SelectBestAdvertisedAddress(underlays, nil) + if underlay == nil { + return underlays + } + return []ma.Multiaddr{underlay} +} diff --git a/pkg/p2p/streamtest/streamtest.go b/pkg/p2p/streamtest/streamtest.go index 3265f6ea566..be491c3d5eb 100644 --- a/pkg/p2p/streamtest/streamtest.go +++ b/pkg/p2p/streamtest/streamtest.go @@ -223,6 +223,12 @@ func (r *Recorder) WaitRecords(t *testing.T, addr swarm.Address, proto, version, return recs } +// IsBee260 implements p2p.Bee260CompatibilityStreamer interface. +// It always returns false. +func (r *Recorder) IsBee260(overlay swarm.Address) bool { + return false +} + type Record struct { in *record out *record From 9f2f7aa7a8e6530e688fbbd8a28a7098d18b7f3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ljubi=C5=A1a=20Ga=C4=8Devi=C4=87?= <35105035+gacevicljubisa@users.noreply.github.com> Date: Thu, 29 Jan 2026 13:07:25 +0100 Subject: [PATCH 75/96] fix(p2p): use connection remote addr as fallback when peerstore is empty (#5336) --- pkg/p2p/libp2p/libp2p.go | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/pkg/p2p/libp2p/libp2p.go b/pkg/p2p/libp2p/libp2p.go index e4edacb3453..367490d9dc0 100644 --- a/pkg/p2p/libp2p/libp2p.go +++ b/pkg/p2p/libp2p/libp2p.go @@ -646,7 +646,7 @@ func (s *Service) handleIncoming(stream network.Stream) { peerID := stream.Conn().RemotePeer() handshakeStream := newStream(stream, s.metrics) - peerMultiaddrs, err := s.peerMultiaddrs(s.ctx, peerID) + peerMultiaddrs, err := s.peerMultiaddrs(s.ctx, stream.Conn().RemoteMultiaddr(), peerID) if err != nil { s.logger.Debug("stream handler: handshake: build remote multiaddrs", "peer_id", peerID, "error", err) s.logger.Error(nil, "stream handler: handshake: build remote multiaddrs", "peer_id", peerID) @@ -1065,7 +1065,7 @@ func (s *Service) Connect(ctx context.Context, addrs []ma.Multiaddr) (address *b handshakeStream := newStream(stream, s.metrics) - peerMultiaddrs, err := s.peerMultiaddrs(ctx, peerID) + peerMultiaddrs, err := s.peerMultiaddrs(ctx, stream.Conn().RemoteMultiaddr(), peerID) if err != nil { _ = handshakeStream.Reset() _ = s.host.Network().ClosePeer(peerID) @@ -1466,13 +1466,18 @@ func (s *Service) determineCurrentNetworkStatus(err error) error { } // peerMultiaddrs builds full multiaddresses for a peer given information from -// libp2p host peerstore and falling back to the remote address from the -// connection. -func (s *Service) peerMultiaddrs(ctx context.Context, peerID libp2ppeer.ID) ([]ma.Multiaddr, error) { +// the libp2p host peerstore. If the peerstore doesn't have addresses yet, +// it falls back to using the remote address from the active connection. +func (s *Service) peerMultiaddrs(ctx context.Context, remoteAddr ma.Multiaddr, peerID libp2ppeer.ID) ([]ma.Multiaddr, error) { waitPeersCtx, cancel := context.WithTimeout(ctx, peerstoreWaitAddrsTimeout) defer cancel() - return buildFullMAs(waitPeerAddrs(waitPeersCtx, s.host.Peerstore(), peerID), peerID) + mas := waitPeerAddrs(waitPeersCtx, s.host.Peerstore(), peerID) + if len(mas) == 0 && remoteAddr != nil { + mas = []ma.Multiaddr{remoteAddr} + } + + return buildFullMAs(mas, peerID) } // IsBee260 implements p2p.Bee260CompatibilityStreamer interface. From 6ddf9b4585f5bce700a5a31c581a044ad1ebeb3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ljubi=C5=A1a=20Ga=C4=8Devi=C4=87?= <35105035+gacevicljubisa@users.noreply.github.com> Date: Thu, 29 Jan 2026 13:07:39 +0100 Subject: [PATCH 76/96] refactor(libp2p): rename underlays to overlayToPeerID in peer registry (#5337) --- pkg/p2p/libp2p/peer.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/p2p/libp2p/peer.go b/pkg/p2p/libp2p/peer.go index 5fcb32303f0..232fbf9b103 100644 --- a/pkg/p2p/libp2p/peer.go +++ b/pkg/p2p/libp2p/peer.go @@ -18,7 +18,7 @@ import ( ) type peerRegistry struct { - underlays map[string]libp2ppeer.ID // map overlay address to underlay peer id + overlayToPeerID map[string]libp2ppeer.ID // map overlay address to underlay peer id overlays map[libp2ppeer.ID]swarm.Address // map underlay peer id to overlay address full map[libp2ppeer.ID]bool // map to track whether a node is full or light node (true=full) bee260Compatibility map[libp2ppeer.ID]bool // map to track bee260 backward compatibility @@ -37,7 +37,7 @@ type disconnecter interface { func newPeerRegistry() *peerRegistry { return &peerRegistry{ - underlays: make(map[string]libp2ppeer.ID), + overlayToPeerID: make(map[string]libp2ppeer.ID), overlays: make(map[libp2ppeer.ID]swarm.Address), full: make(map[libp2ppeer.ID]bool), bee260Compatibility: make(map[libp2ppeer.ID]bool), @@ -77,7 +77,7 @@ func (r *peerRegistry) Disconnected(_ network.Network, c network.Conn) { delete(r.connections, peerID) overlay := r.overlays[peerID] delete(r.overlays, peerID) - delete(r.underlays, overlay.ByteString()) + delete(r.overlayToPeerID, overlay.ByteString()) for _, cancel := range r.streams[peerID] { cancel() } @@ -146,12 +146,12 @@ func (r *peerRegistry) addIfNotExists(c network.Conn, overlay swarm.Address, ful // this is solving a case of multiple underlying libp2p connections for the same peer r.connections[peerID][c] = struct{}{} - if _, exists := r.underlays[overlay.ByteString()]; exists { + if _, exists := r.overlayToPeerID[overlay.ByteString()]; exists { return true } r.streams[peerID] = make(map[network.Stream]context.CancelFunc) - r.underlays[overlay.ByteString()] = peerID + r.overlayToPeerID[overlay.ByteString()] = peerID r.overlays[peerID] = overlay r.full[peerID] = full return false @@ -160,7 +160,7 @@ func (r *peerRegistry) addIfNotExists(c network.Conn, overlay swarm.Address, ful func (r *peerRegistry) peerID(overlay swarm.Address) (peerID libp2ppeer.ID, found bool) { r.mu.RLock() - peerID, found = r.underlays[overlay.ByteString()] + peerID, found = r.overlayToPeerID[overlay.ByteString()] r.mu.RUnlock() return peerID, found } @@ -223,9 +223,9 @@ func (r *peerRegistry) isConnected(peerID libp2ppeer.ID, remoteAddr ma.Multiaddr func (r *peerRegistry) remove(overlay swarm.Address) (found, full bool, peerID libp2ppeer.ID) { r.mu.Lock() - peerID, found = r.underlays[overlay.ByteString()] + peerID, found = r.overlayToPeerID[overlay.ByteString()] delete(r.overlays, peerID) - delete(r.underlays, overlay.ByteString()) + delete(r.overlayToPeerID, overlay.ByteString()) delete(r.connections, peerID) for _, cancel := range r.streams[peerID] { cancel() From 60ca478c496a8952c5e25394d75f99a3cf1a734c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ljubi=C5=A1a=20Ga=C4=8Devi=C4=87?= <35105035+gacevicljubisa@users.noreply.github.com> Date: Tue, 10 Feb 2026 11:00:42 +0100 Subject: [PATCH 77/96] chore: bump open api version to 7.4.0 (#5353) --- openapi/Swarm.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openapi/Swarm.yaml b/openapi/Swarm.yaml index ab8c7a47c24..f03aa20a1c2 100644 --- a/openapi/Swarm.yaml +++ b/openapi/Swarm.yaml @@ -1,7 +1,7 @@ openapi: 3.0.3 info: - version: 7.3.0 + version: 7.4.0 title: Bee API description: "A list of the currently provided Interfaces to interact with the swarm, implementing file operations and sending messages" From 3f6c6c9356da9f6a6cf6a5a09649c9686977ca2f Mon Sep 17 00:00:00 2001 From: Akrem Chabchoub <121046693+akrem-chabchoub@users.noreply.github.com> Date: Wed, 11 Feb 2026 13:52:58 +0100 Subject: [PATCH 78/96] =?UTF-8?q?fix(workflows):=20add=20failure=20checks?= =?UTF-8?q?=20for=20gsoc,=20postage-stamps,=20stake,=20w=E2=80=A6=20(#5354?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/beekeeper.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/beekeeper.yml b/.github/workflows/beekeeper.yml index 43ac9b18c37..c2448465d7f 100644 --- a/.github/workflows/beekeeper.yml +++ b/.github/workflows/beekeeper.yml @@ -192,11 +192,16 @@ jobs: if ${{ steps.settlements.outcome=='failure' }}; then FAILED=settlements; fi if ${{ steps.pss.outcome=='failure' }}; then FAILED=pss; fi if ${{ steps.soc.outcome=='failure' }}; then FAILED=soc; fi + if ${{ steps.gsoc.outcome=='failure' }}; then FAILED=gsoc; fi if ${{ steps.pushsync-chunks-1.outcome=='failure' }}; then FAILED=pushsync-chunks-1; fi if ${{ steps.pushsync-chunks-2.outcome=='failure' }}; then FAILED=pushsync-chunks-2; fi if ${{ steps.retrieval.outcome=='failure' }}; then FAILED=retrieval; fi if ${{ steps.manifest.outcome=='failure' }}; then FAILED=manifest; fi if ${{ steps.manifest-v1.outcome=='failure' }}; then FAILED=manifest-v1; fi + if ${{ steps.postage-stamps.outcome=='failure' }}; then FAILED=postage-stamps; fi + if ${{ steps.stake.outcome=='failure' }}; then FAILED=stake; fi + if ${{ steps.withdraw.outcome=='failure' }}; then FAILED=withdraw; fi + if ${{ steps.redundancy.outcome=='failure' }}; then FAILED=redundancy; fi if ${{ steps.feeds.outcome=='failure' }}; then FAILED=feeds; fi if ${{ steps.feeds-v1.outcome=='failure' }}; then FAILED=feeds-v1; fi if ${{ steps.act.outcome=='failure' }}; then FAILED=act; fi From 91b01ad24e5e99336e0473d3a86364279801d59c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ljubi=C5=A1a=20Ga=C4=8Devi=C4=87?= <35105035+gacevicljubisa@users.noreply.github.com> Date: Fri, 13 Feb 2026 16:39:53 +0100 Subject: [PATCH 79/96] fix(p2p): give each underlay address fresh 15s connection timeout (#5328) --- pkg/p2p/libp2p/libp2p.go | 7 ++++++- pkg/topology/kademlia/kademlia.go | 14 +++++++------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/pkg/p2p/libp2p/libp2p.go b/pkg/p2p/libp2p/libp2p.go index 367490d9dc0..9765268754b 100644 --- a/pkg/p2p/libp2p/libp2p.go +++ b/pkg/p2p/libp2p/libp2p.go @@ -999,6 +999,7 @@ func (s *Service) Connect(ctx context.Context, addrs []ma.Multiaddr) (address *b // addresses for the same peer is passed to the host.Host.Connect function // and reachabiltiy Private is emitted on libp2p EventBus(), which results // in weaker connectivity and failures in some integration tests. + for _, addr := range addrs { // Extract the peer ID from the multiaddr. ai, err := libp2ppeer.AddrInfoFromP2pAddr(addr) @@ -1031,7 +1032,11 @@ func (s *Service) Connect(ctx context.Context, addrs []ma.Multiaddr) (address *b return address, p2p.ErrAlreadyConnected } - if err := s.connectionBreaker.Execute(func() error { return s.host.Connect(ctx, *info) }); err != nil { + connectCtx, cancel := context.WithTimeout(ctx, 15*time.Second) + err = s.connectionBreaker.Execute(func() error { return s.host.Connect(connectCtx, *info) }) + cancel() + + if err != nil { if errors.Is(err, breaker.ErrClosed) { s.metrics.ConnectBreakerCount.Inc() return nil, p2p.NewConnectionBackoffError(err, s.connectionBreaker.ClosedUntil()) diff --git a/pkg/topology/kademlia/kademlia.go b/pkg/topology/kademlia/kademlia.go index b31dd6d6c6d..d4b28e8eb1b 100644 --- a/pkg/topology/kademlia/kademlia.go +++ b/pkg/topology/kademlia/kademlia.go @@ -42,10 +42,9 @@ const ( addPeerBatchSize = 500 - // To avoid context.Timeout errors during network failure, the value of - // the peerConnectionAttemptTimeout constant must be equal to or greater - // than 5 seconds (empirically verified). - peerConnectionAttemptTimeout = 15 * time.Second // timeout for establishing a new connection with peer. + // Each underlay address gets up to 15s for connection (in libp2p.Connect). + // This budget allows multiple addresses to be tried sequentially per peer. + peerConnectionAttemptTimeout = 45 * time.Second // timeout for establishing a new connection with peer. ) // Default option values @@ -807,9 +806,6 @@ func (k *Kad) connectBootNodes(ctx context.Context) { var attempts, connected int totalAttempts := maxBootNodeAttempts * len(k.opt.Bootnodes) - ctx, cancel := context.WithTimeout(ctx, 15*time.Second) - defer cancel() - for _, addr := range k.opt.Bootnodes { if attempts >= totalAttempts || connected >= 3 { return @@ -820,6 +816,10 @@ func (k *Kad) connectBootNodes(ctx context.Context) { if attempts >= maxBootNodeAttempts { return true, nil } + + ctx, cancel := context.WithTimeout(ctx, peerConnectionAttemptTimeout) + defer cancel() + bzzAddress, err := k.p2p.Connect(ctx, []ma.Multiaddr{addr}) attempts++ From 668bd8bc4b9597a32cbcd93a61b357ec023bdcb8 Mon Sep 17 00:00:00 2001 From: Akrem Chabchoub <121046693+akrem-chabchoub@users.noreply.github.com> Date: Fri, 13 Feb 2026 18:07:01 +0100 Subject: [PATCH 80/96] =?UTF-8?q?fix(postage):=20prevent=20nil=20pointer?= =?UTF-8?q?=20panic=20in=20TopUpBatch=20on=20transaction=20=E2=80=A6=20(#5?= =?UTF-8?q?332)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/postage/postagecontract/contract.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/postage/postagecontract/contract.go b/pkg/postage/postagecontract/contract.go index d9645822166..e8570b64e9b 100644 --- a/pkg/postage/postagecontract/contract.go +++ b/pkg/postage/postagecontract/contract.go @@ -390,7 +390,7 @@ func (c *postageContract) CreateBatch(ctx context.Context, initialBalance *big.I if err != nil { return } - txHash = receipt.TxHash + for _, ev := range receipt.Logs { if ev.Address == c.postageStampContractAddress && len(ev.Topics) > 0 && ev.Topics[0] == c.batchCreatedTopic { var createdEvent batchCreatedEvent @@ -415,6 +415,7 @@ func (c *postageContract) CreateBatch(ctx context.Context, initialBalance *big.I if err != nil { return } + txHash = receipt.TxHash return } } @@ -447,7 +448,6 @@ func (c *postageContract) TopUpBatch(ctx context.Context, batchID []byte, topupB receipt, err := c.sendTopUpBatchTransaction(ctx, batch.ID, topupBalance) if err != nil { - txHash = receipt.TxHash return } @@ -483,9 +483,10 @@ func (c *postageContract) DiluteBatch(ctx context.Context, batchID []byte, newDe if err != nil { return } - txHash = receipt.TxHash + for _, ev := range receipt.Logs { if ev.Address == c.postageStampContractAddress && len(ev.Topics) > 0 && ev.Topics[0] == c.batchDepthIncreaseTopic { + txHash = receipt.TxHash return } } From 579f77638979d7d03a33cff3ae1f4e628eb5e659 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ljubi=C5=A1a=20Ga=C4=8Devi=C4=87?= <35105035+gacevicljubisa@users.noreply.github.com> Date: Fri, 13 Feb 2026 18:15:32 +0100 Subject: [PATCH 81/96] fix(p2p): improve reacher address selection and transport filtering (#5347) --- pkg/bzz/address.go | 34 --- pkg/bzz/address_test.go | 128 -------- pkg/bzz/transport.go | 108 +++++++ pkg/bzz/transport_test.go | 338 +++++++++++++++++++++ pkg/p2p/error.go | 2 + pkg/p2p/libp2p/export_test.go | 14 + pkg/p2p/libp2p/filter_addresses_test.go | 218 +++++++++++++ pkg/p2p/libp2p/internal/reacher/reacher.go | 5 +- pkg/p2p/libp2p/libp2p.go | 71 +++-- pkg/topology/kademlia/kademlia.go | 15 +- 10 files changed, 742 insertions(+), 191 deletions(-) create mode 100644 pkg/bzz/transport.go create mode 100644 pkg/bzz/transport_test.go create mode 100644 pkg/p2p/libp2p/filter_addresses_test.go diff --git a/pkg/bzz/address.go b/pkg/bzz/address.go index 1a4021e9eff..89914ef66bf 100644 --- a/pkg/bzz/address.go +++ b/pkg/bzz/address.go @@ -15,14 +15,12 @@ import ( "errors" "fmt" "slices" - "sort" "github.com/ethereum/go-ethereum/common" "github.com/ethersphere/bee/v2/pkg/crypto" "github.com/ethersphere/bee/v2/pkg/swarm" ma "github.com/multiformats/go-multiaddr" - manet "github.com/multiformats/go-multiaddr/net" ) var ErrInvalidAddress = errors.New("invalid address") @@ -222,35 +220,3 @@ func parseMultiaddrs(addrs []string) ([]ma.Multiaddr, error) { } return multiAddrs, nil } - -func SelectBestAdvertisedAddress(addrs []ma.Multiaddr, fallback ma.Multiaddr) ma.Multiaddr { - if len(addrs) == 0 { - return fallback - } - - hasTCPProtocol := func(addr ma.Multiaddr) bool { - _, err := addr.ValueForProtocol(ma.P_TCP) - return err == nil - } - - // Sort addresses to prioritize TCP over other protocols - sort.SliceStable(addrs, func(i, j int) bool { - iTCP := hasTCPProtocol(addrs[i]) - jTCP := hasTCPProtocol(addrs[j]) - return iTCP && !jTCP - }) - - for _, addr := range addrs { - if manet.IsPublicAddr(addr) { - return addr - } - } - - for _, addr := range addrs { - if !manet.IsPrivateAddr(addr) { - return addr - } - } - - return addrs[0] -} diff --git a/pkg/bzz/address_test.go b/pkg/bzz/address_test.go index 0c50b08de64..8d038010e25 100644 --- a/pkg/bzz/address_test.go +++ b/pkg/bzz/address_test.go @@ -64,134 +64,6 @@ func TestBzzAddress(t *testing.T) { } } -func TestSelectBestAdvertisedAddress(t *testing.T) { - t.Parallel() - - mustMultiaddr := func(s string) multiaddr.Multiaddr { - addr, err := multiaddr.NewMultiaddr(s) - if err != nil { - t.Fatalf("failed to create multiaddr %s: %v", s, err) - } - return addr - } - - tests := []struct { - name string - addrs []multiaddr.Multiaddr - fallback multiaddr.Multiaddr - expected multiaddr.Multiaddr - }{ - { - name: "empty addresses returns fallback", - addrs: []multiaddr.Multiaddr{}, - fallback: mustMultiaddr("/ip4/127.0.0.1/tcp/8080"), - expected: mustMultiaddr("/ip4/127.0.0.1/tcp/8080"), - }, - { - name: "nil addresses returns fallback", - addrs: nil, - fallback: mustMultiaddr("/ip4/127.0.0.1/tcp/8080"), - expected: mustMultiaddr("/ip4/127.0.0.1/tcp/8080"), - }, - { - name: "prefers public addresses", - addrs: []multiaddr.Multiaddr{ - mustMultiaddr("/ip4/192.168.1.1/tcp/8080"), // private - mustMultiaddr("/ip4/8.8.8.8/tcp/8080"), // public - mustMultiaddr("/ip4/10.0.0.1/tcp/8080"), // private - }, - fallback: mustMultiaddr("/ip4/127.0.0.1/tcp/8080"), - expected: mustMultiaddr("/ip4/8.8.8.8/tcp/8080"), - }, - { - name: "prefers first public address when multiple exist", - addrs: []multiaddr.Multiaddr{ - mustMultiaddr("/ip4/192.168.1.1/tcp/8080"), // private - mustMultiaddr("/ip4/8.8.8.8/tcp/8080"), // public - mustMultiaddr("/ip4/1.1.1.1/tcp/8080"), // public - }, - fallback: mustMultiaddr("/ip4/127.0.0.1/tcp/8080"), - expected: mustMultiaddr("/ip4/8.8.8.8/tcp/8080"), - }, - { - name: "prefers non-private when no public addresses", - addrs: []multiaddr.Multiaddr{ - mustMultiaddr("/ip4/127.0.0.1/tcp/8080"), // loopback - mustMultiaddr("/ip4/192.168.1.1/tcp/8080"), // private but not loopback - mustMultiaddr("/ip4/10.0.0.1/tcp/8080"), // private but not loopback - }, - fallback: mustMultiaddr("/ip4/127.0.0.1/tcp/9999"), - expected: mustMultiaddr("/ip4/127.0.0.1/tcp/8080"), - }, - { - name: "returns first address when all are loopback", - addrs: []multiaddr.Multiaddr{ - mustMultiaddr("/ip4/127.0.0.1/tcp/8080"), - mustMultiaddr("/ip4/127.0.0.1/tcp/8081"), - mustMultiaddr("/ip6/::1/tcp/8080"), - }, - fallback: mustMultiaddr("/ip4/127.0.0.1/tcp/9999"), - expected: mustMultiaddr("/ip4/127.0.0.1/tcp/8080"), - }, - { - name: "sorts TCP addresses first", - addrs: []multiaddr.Multiaddr{ - mustMultiaddr("/ip4/192.168.1.1/udp/8080"), // UDP - mustMultiaddr("/ip4/1.1.1.1/udp/8080"), // UDP public - mustMultiaddr("/ip4/8.8.8.8/tcp/8080"), // TCP public - }, - fallback: mustMultiaddr("/ip4/127.0.0.1/tcp/9999"), - expected: mustMultiaddr("/ip4/8.8.8.8/tcp/8080"), - }, - { - name: "handles IPv6 addresses", - addrs: []multiaddr.Multiaddr{ - mustMultiaddr("/ip6/::1/tcp/8080"), // loopback - mustMultiaddr("/ip6/2001:db8::1/tcp/8080"), // public IPv6 - mustMultiaddr("/ip4/192.168.1.1/tcp/8080"), // private IPv4 - }, - fallback: mustMultiaddr("/ip4/127.0.0.1/tcp/9999"), - expected: mustMultiaddr("/ip6/2001:db8::1/tcp/8080"), - }, - { - name: "handles mixed protocols with preference order", - addrs: []multiaddr.Multiaddr{ - mustMultiaddr("/ip4/192.168.1.1/udp/8080"), // private UDP - mustMultiaddr("/ip4/192.168.1.2/tcp/8080"), // private TCP - mustMultiaddr("/ip4/127.0.0.1/tcp/8080"), // loopback TCP - }, - fallback: mustMultiaddr("/ip4/127.0.0.1/tcp/9999"), - expected: mustMultiaddr("/ip4/192.168.1.2/tcp/8080"), // first TCP, and it's non-loopback - }, - { - name: "single address", - addrs: []multiaddr.Multiaddr{ - mustMultiaddr("/ip4/192.168.1.1/tcp/8080"), - }, - fallback: mustMultiaddr("/ip4/127.0.0.1/tcp/9999"), - expected: mustMultiaddr("/ip4/192.168.1.1/tcp/8080"), - }, - { - name: "websocket addresses", - addrs: []multiaddr.Multiaddr{ - mustMultiaddr("/ip4/127.0.0.1/tcp/8080/ws"), - mustMultiaddr("/ip4/8.8.8.8/tcp/8080/ws"), // public with websocket - }, - fallback: mustMultiaddr("/ip4/127.0.0.1/tcp/9999"), - expected: mustMultiaddr("/ip4/8.8.8.8/tcp/8080/ws"), - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := bzz.SelectBestAdvertisedAddress(tt.addrs, tt.fallback) - if !result.Equal(tt.expected) { - t.Errorf("SelectBestAdvertisedAddress() = %v, want %v", result, tt.expected) - } - }) - } -} - func TestAreUnderlaysEqual(t *testing.T) { // --- Test Data Initialization --- addr1 := mustNewMultiaddr(t, "/ip4/127.0.0.1/tcp/8001") diff --git a/pkg/bzz/transport.go b/pkg/bzz/transport.go new file mode 100644 index 00000000000..579a1807936 --- /dev/null +++ b/pkg/bzz/transport.go @@ -0,0 +1,108 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bzz + +import ( + "sort" + + ma "github.com/multiformats/go-multiaddr" + manet "github.com/multiformats/go-multiaddr/net" +) + +// TransportType represents the transport protocol of a multiaddress. +type TransportType int + +const ( + // TransportUnknown indicates an unrecognized transport. + TransportUnknown TransportType = iota + // TransportTCP indicates plain TCP without WebSocket. + TransportTCP + // TransportWS indicates WebSocket without TLS. + TransportWS + // TransportWSS indicates WebSocket with TLS (secure). + TransportWSS +) + +// String returns a string representation of the transport type. +func (t TransportType) String() string { + switch t { + case TransportTCP: + return "tcp" + case TransportWS: + return "ws" + case TransportWSS: + return "wss" + default: + return "unknown" + } +} + +// Priority returns the sorting priority for the transport type. +// Lower value = higher priority: TCP (0) > WS (1) > WSS (2) > Unknown (3) +func (t TransportType) Priority() int { + switch t { + case TransportTCP: + return 0 + case TransportWS: + return 1 + case TransportWSS: + return 2 + default: + return 3 + } +} + +// ClassifyTransport returns the transport type of a multiaddress. +// It distinguishes between plain TCP, WebSocket (WS), and secure WebSocket (WSS). +func ClassifyTransport(addr ma.Multiaddr) TransportType { + if addr == nil { + return TransportUnknown + } + + hasProtocol := func(p int) bool { + _, err := addr.ValueForProtocol(p) + return err == nil + } + + hasWS := hasProtocol(ma.P_WS) + hasTLS := hasProtocol(ma.P_TLS) + hasTCP := hasProtocol(ma.P_TCP) + + switch { + case hasWS && hasTLS: + return TransportWSS + case hasWS: + return TransportWS + case hasTCP: + return TransportTCP + default: + return TransportUnknown + } +} + +func SelectBestAdvertisedAddress(addrs []ma.Multiaddr, fallback ma.Multiaddr) ma.Multiaddr { + if len(addrs) == 0 { + return fallback + } + + // Sort addresses: first by transport priority (TCP > WS > WSS), preserving relative order + sort.SliceStable(addrs, func(i, j int) bool { + return ClassifyTransport(addrs[i]).Priority() < ClassifyTransport(addrs[j]).Priority() + }) + + for _, addr := range addrs { + if manet.IsPublicAddr(addr) { + return addr + } + } + + for _, addr := range addrs { + if !manet.IsPrivateAddr(addr) { + return addr + } + } + + return addrs[0] +} diff --git a/pkg/bzz/transport_test.go b/pkg/bzz/transport_test.go new file mode 100644 index 00000000000..fab5f90246f --- /dev/null +++ b/pkg/bzz/transport_test.go @@ -0,0 +1,338 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bzz_test + +import ( + "testing" + + "github.com/ethersphere/bee/v2/pkg/bzz" + + "github.com/multiformats/go-multiaddr" +) + +func TestSelectBestAdvertisedAddress(t *testing.T) { + t.Parallel() + + mustMultiaddr := func(s string) multiaddr.Multiaddr { + addr, err := multiaddr.NewMultiaddr(s) + if err != nil { + t.Fatalf("failed to create multiaddr %s: %v", s, err) + } + return addr + } + + tests := []struct { + name string + addrs []multiaddr.Multiaddr + fallback multiaddr.Multiaddr + expected multiaddr.Multiaddr + }{ + { + name: "empty addresses returns fallback", + addrs: []multiaddr.Multiaddr{}, + fallback: mustMultiaddr("/ip4/127.0.0.1/tcp/8080"), + expected: mustMultiaddr("/ip4/127.0.0.1/tcp/8080"), + }, + { + name: "nil addresses returns fallback", + addrs: nil, + fallback: mustMultiaddr("/ip4/127.0.0.1/tcp/8080"), + expected: mustMultiaddr("/ip4/127.0.0.1/tcp/8080"), + }, + { + name: "prefers public addresses", + addrs: []multiaddr.Multiaddr{ + mustMultiaddr("/ip4/192.168.1.1/tcp/8080"), // private + mustMultiaddr("/ip4/8.8.8.8/tcp/8080"), // public + mustMultiaddr("/ip4/10.0.0.1/tcp/8080"), // private + }, + fallback: mustMultiaddr("/ip4/127.0.0.1/tcp/8080"), + expected: mustMultiaddr("/ip4/8.8.8.8/tcp/8080"), + }, + { + name: "prefers first public address when multiple exist", + addrs: []multiaddr.Multiaddr{ + mustMultiaddr("/ip4/192.168.1.1/tcp/8080"), // private + mustMultiaddr("/ip4/8.8.8.8/tcp/8080"), // public + mustMultiaddr("/ip4/1.1.1.1/tcp/8080"), // public + }, + fallback: mustMultiaddr("/ip4/127.0.0.1/tcp/8080"), + expected: mustMultiaddr("/ip4/8.8.8.8/tcp/8080"), + }, + { + name: "prefers non-private when no public addresses", + addrs: []multiaddr.Multiaddr{ + mustMultiaddr("/ip4/127.0.0.1/tcp/8080"), // loopback + mustMultiaddr("/ip4/192.168.1.1/tcp/8080"), // private but not loopback + mustMultiaddr("/ip4/10.0.0.1/tcp/8080"), // private but not loopback + }, + fallback: mustMultiaddr("/ip4/127.0.0.1/tcp/9999"), + expected: mustMultiaddr("/ip4/127.0.0.1/tcp/8080"), + }, + { + name: "returns first address when all are loopback", + addrs: []multiaddr.Multiaddr{ + mustMultiaddr("/ip4/127.0.0.1/tcp/8080"), + mustMultiaddr("/ip4/127.0.0.1/tcp/8081"), + mustMultiaddr("/ip6/::1/tcp/8080"), + }, + fallback: mustMultiaddr("/ip4/127.0.0.1/tcp/9999"), + expected: mustMultiaddr("/ip4/127.0.0.1/tcp/8080"), + }, + { + name: "sorts TCP addresses first", + addrs: []multiaddr.Multiaddr{ + mustMultiaddr("/ip4/192.168.1.1/udp/8080"), // UDP + mustMultiaddr("/ip4/1.1.1.1/udp/8080"), // UDP public + mustMultiaddr("/ip4/8.8.8.8/tcp/8080"), // TCP public + }, + fallback: mustMultiaddr("/ip4/127.0.0.1/tcp/9999"), + expected: mustMultiaddr("/ip4/8.8.8.8/tcp/8080"), + }, + { + name: "handles IPv6 addresses", + addrs: []multiaddr.Multiaddr{ + mustMultiaddr("/ip6/::1/tcp/8080"), // loopback + mustMultiaddr("/ip6/2001:db8::1/tcp/8080"), // public IPv6 + mustMultiaddr("/ip4/192.168.1.1/tcp/8080"), // private IPv4 + }, + fallback: mustMultiaddr("/ip4/127.0.0.1/tcp/9999"), + expected: mustMultiaddr("/ip6/2001:db8::1/tcp/8080"), + }, + { + name: "handles mixed protocols with preference order", + addrs: []multiaddr.Multiaddr{ + mustMultiaddr("/ip4/192.168.1.1/udp/8080"), // private UDP + mustMultiaddr("/ip4/192.168.1.2/tcp/8080"), // private TCP + mustMultiaddr("/ip4/127.0.0.1/tcp/8080"), // loopback TCP + }, + fallback: mustMultiaddr("/ip4/127.0.0.1/tcp/9999"), + expected: mustMultiaddr("/ip4/192.168.1.2/tcp/8080"), // first TCP, and it's non-loopback + }, + { + name: "single address", + addrs: []multiaddr.Multiaddr{ + mustMultiaddr("/ip4/192.168.1.1/tcp/8080"), + }, + fallback: mustMultiaddr("/ip4/127.0.0.1/tcp/9999"), + expected: mustMultiaddr("/ip4/192.168.1.1/tcp/8080"), + }, + { + name: "websocket addresses", + addrs: []multiaddr.Multiaddr{ + mustMultiaddr("/ip4/127.0.0.1/tcp/8080/ws"), + mustMultiaddr("/ip4/8.8.8.8/tcp/8080/ws"), // public with websocket + }, + fallback: mustMultiaddr("/ip4/127.0.0.1/tcp/9999"), + expected: mustMultiaddr("/ip4/8.8.8.8/tcp/8080/ws"), + }, + // Full underlay addresses tests + { + name: "full underlay addresses: prefers public TCP over private TCP", + addrs: []multiaddr.Multiaddr{ + mustMultiaddr("/ip4/10.233.99.120/tcp/1634/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), // private + mustMultiaddr("/ip4/104.28.194.73/tcp/32002/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), // public + mustMultiaddr("/ip4/127.0.0.1/tcp/1634/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), // loopback + }, + fallback: nil, + expected: mustMultiaddr("/ip4/104.28.194.73/tcp/32002/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), + }, + { + name: "full underlay addresses: prefers public TCP over WSS addresses", + addrs: []multiaddr.Multiaddr{ + mustMultiaddr("/ip4/10.233.99.120/tcp/1635/tls/sni/10-233-99-120.k2k4r8pr3m3aug5nudg2y039qfj2gxw6wnlx0e0ghzxufcn38soyp9z4.libp2p.direct/ws/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), // private WSS + mustMultiaddr("/ip4/104.28.194.73/tcp/32002/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), // public TCP + mustMultiaddr("/ip4/104.28.194.73/tcp/32532/tls/sni/104-28-194-73.k2k4r8pr3m3aug5nudg2y039qfj2gxw6wnlx0e0ghzxufcn38soyp9z4.libp2p.direct/ws/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), // public WSS + }, + fallback: nil, + expected: mustMultiaddr("/ip4/104.28.194.73/tcp/32002/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), + }, + { + name: "full underlay addresses: full node underlay list selects public TCP", + addrs: []multiaddr.Multiaddr{ + mustMultiaddr("/ip4/10.233.99.120/tcp/1634/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), + mustMultiaddr("/ip4/10.233.99.120/tcp/1635/tls/sni/10-233-99-120.k2k4r8pr3m3aug5nudg2y039qfj2gxw6wnlx0e0ghzxufcn38soyp9z4.libp2p.direct/ws/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), + mustMultiaddr("/ip4/104.28.194.73/tcp/32002/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), + mustMultiaddr("/ip4/104.28.194.73/tcp/32532/tls/sni/104-28-194-73.k2k4r8pr3m3aug5nudg2y039qfj2gxw6wnlx0e0ghzxufcn38soyp9z4.libp2p.direct/ws/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), + mustMultiaddr("/ip4/127.0.0.1/tcp/1634/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), + mustMultiaddr("/ip4/127.0.0.1/tcp/1635/tls/sni/127-0-0-1.k2k4r8pr3m3aug5nudg2y039qfj2gxw6wnlx0e0ghzxufcn38soyp9z4.libp2p.direct/ws/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), + mustMultiaddr("/ip6/::1/tcp/1634/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), + mustMultiaddr("/ip6/::1/tcp/1635/tls/sni/0--1.k2k4r8pr3m3aug5nudg2y039qfj2gxw6wnlx0e0ghzxufcn38soyp9z4.libp2p.direct/ws/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), + }, + fallback: nil, + expected: mustMultiaddr("/ip4/104.28.194.73/tcp/32002/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), + }, + { + name: "full underlay addresses: WSS only list selects public WSS", + addrs: []multiaddr.Multiaddr{ + mustMultiaddr("/ip4/10.233.99.120/tcp/1635/tls/sni/10-233-99-120.k2k4r8pr3m3aug5nudg2y039qfj2gxw6wnlx0e0ghzxufcn38soyp9z4.libp2p.direct/ws/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), + mustMultiaddr("/ip4/104.28.194.73/tcp/32532/tls/sni/104-28-194-73.k2k4r8pr3m3aug5nudg2y039qfj2gxw6wnlx0e0ghzxufcn38soyp9z4.libp2p.direct/ws/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), + mustMultiaddr("/ip4/127.0.0.1/tcp/1635/tls/sni/127-0-0-1.k2k4r8pr3m3aug5nudg2y039qfj2gxw6wnlx0e0ghzxufcn38soyp9z4.libp2p.direct/ws/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), + mustMultiaddr("/ip6/::1/tcp/1635/tls/sni/0--1.k2k4r8pr3m3aug5nudg2y039qfj2gxw6wnlx0e0ghzxufcn38soyp9z4.libp2p.direct/ws/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), + }, + fallback: nil, + expected: mustMultiaddr("/ip4/104.28.194.73/tcp/32532/tls/sni/104-28-194-73.k2k4r8pr3m3aug5nudg2y039qfj2gxw6wnlx0e0ghzxufcn38soyp9z4.libp2p.direct/ws/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), + }, + { + name: "full underlay addresses: TCP only list selects public TCP", + addrs: []multiaddr.Multiaddr{ + mustMultiaddr("/ip4/10.233.99.120/tcp/1634/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), + mustMultiaddr("/ip4/104.28.194.73/tcp/32002/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), + mustMultiaddr("/ip4/127.0.0.1/tcp/1634/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), + mustMultiaddr("/ip6/::1/tcp/1634/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), + }, + fallback: nil, + expected: mustMultiaddr("/ip4/104.28.194.73/tcp/32002/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), + }, + { + name: "full underlay addresses: IPv6 loopback list with no public addresses", + addrs: []multiaddr.Multiaddr{ + mustMultiaddr("/ip6/::1/tcp/1634/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), + mustMultiaddr("/ip6/::1/tcp/1635/tls/sni/0--1.k2k4r8pr3m3aug5nudg2y039qfj2gxw6wnlx0e0ghzxufcn38soyp9z4.libp2p.direct/ws/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), + }, + fallback: nil, + expected: mustMultiaddr("/ip6/::1/tcp/1634/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), + }, + { + name: "full underlay addresses: WSS before TCP in input - TCP is still selected due to transport priority", + addrs: []multiaddr.Multiaddr{ + mustMultiaddr("/ip4/104.28.194.73/tcp/32532/tls/sni/104-28-194-73.k2k4r8pr3m3aug5nudg2y039qfj2gxw6wnlx0e0ghzxufcn38soyp9z4.libp2p.direct/ws/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), // public WSS (first in input) + mustMultiaddr("/ip4/104.28.194.73/tcp/32002/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), // public TCP (second in input) + }, + fallback: nil, + // TCP is selected because: + // 1. Transport priority: TCP (0) > WS (1) > WSS (2) + // 2. Sorting puts TCP addresses before WSS addresses + // 3. Both are public, so the first public after sorting (TCP) is returned + expected: mustMultiaddr("/ip4/104.28.194.73/tcp/32002/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), + }, + { + name: "full underlay addresses: TCP before WSS in input - TCP is selected", + addrs: []multiaddr.Multiaddr{ + mustMultiaddr("/ip4/104.28.194.73/tcp/32002/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), // public TCP (first) + mustMultiaddr("/ip4/104.28.194.73/tcp/32532/tls/sni/104-28-194-73.k2k4r8pr3m3aug5nudg2y039qfj2gxw6wnlx0e0ghzxufcn38soyp9z4.libp2p.direct/ws/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), // public WSS (second) + }, + fallback: nil, + // TCP is selected because it has higher transport priority + expected: mustMultiaddr("/ip4/104.28.194.73/tcp/32002/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), + }, + { + name: "full underlay addresses: private TCP vs public WSS - public WSS is selected", + addrs: []multiaddr.Multiaddr{ + mustMultiaddr("/ip4/10.233.99.120/tcp/1634/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), // private TCP + mustMultiaddr("/ip4/104.28.194.73/tcp/32532/tls/sni/104-28-194-73.k2k4r8pr3m3aug5nudg2y039qfj2gxw6wnlx0e0ghzxufcn38soyp9z4.libp2p.direct/ws/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), // public WSS + }, + fallback: nil, + // Public WSS is selected because there is no public TCP address + // Priority: public addresses > private addresses, then by transport type + expected: mustMultiaddr("/ip4/104.28.194.73/tcp/32532/tls/sni/104-28-194-73.k2k4r8pr3m3aug5nudg2y039qfj2gxw6wnlx0e0ghzxufcn38soyp9z4.libp2p.direct/ws/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := bzz.SelectBestAdvertisedAddress(tt.addrs, tt.fallback) + if !result.Equal(tt.expected) { + t.Errorf("SelectBestAdvertisedAddress() = %v, want %v", result, tt.expected) + } + }) + } +} + +func TestClassifyTransport(t *testing.T) { + t.Parallel() + + mustMultiaddr := func(s string) multiaddr.Multiaddr { + addr, err := multiaddr.NewMultiaddr(s) + if err != nil { + t.Fatalf("failed to create multiaddr %s: %v", s, err) + } + return addr + } + + tests := []struct { + name string + addr multiaddr.Multiaddr + expected bzz.TransportType + }{ + { + name: "plain TCP address", + addr: mustMultiaddr("/ip4/104.28.194.73/tcp/32002/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), + expected: bzz.TransportTCP, + }, + { + name: "plain TCP IPv6 address", + addr: mustMultiaddr("/ip6/::1/tcp/1634/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), + expected: bzz.TransportTCP, + }, + { + name: "plain WS address", + addr: mustMultiaddr("/ip4/127.0.0.1/tcp/8080/ws/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), + expected: bzz.TransportWS, + }, + { + name: "WSS address with TLS and SNI", + addr: mustMultiaddr("/ip4/104.28.194.73/tcp/32532/tls/sni/104-28-194-73.k2k4r8pr3m3aug5nudg2y039qfj2gxw6wnlx0e0ghzxufcn38soyp9z4.libp2p.direct/ws/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), + expected: bzz.TransportWSS, + }, + { + name: "WSS IPv6 address", + addr: mustMultiaddr("/ip6/::1/tcp/1635/tls/sni/0--1.k2k4r8pr3m3aug5nudg2y039qfj2gxw6wnlx0e0ghzxufcn38soyp9z4.libp2p.direct/ws/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj"), + expected: bzz.TransportWSS, + }, + { + name: "UDP address returns unknown", + addr: mustMultiaddr("/ip4/127.0.0.1/udp/8080"), + expected: bzz.TransportUnknown, + }, + { + name: "nil address returns unknown", + addr: nil, + expected: bzz.TransportUnknown, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := bzz.ClassifyTransport(tt.addr) + if result != tt.expected { + t.Errorf("ClassifyTransport() = %v (%s), want %v (%s)", result, result.String(), tt.expected, tt.expected.String()) + } + }) + } +} + +func TestTransportTypePriority(t *testing.T) { + t.Parallel() + + tests := []struct { + transport bzz.TransportType + priority int + }{ + {bzz.TransportTCP, 0}, + {bzz.TransportWS, 1}, + {bzz.TransportWSS, 2}, + {bzz.TransportUnknown, 3}, + } + + for _, tt := range tests { + t.Run(tt.transport.String(), func(t *testing.T) { + if got := tt.transport.Priority(); got != tt.priority { + t.Errorf("TransportType(%v).Priority() = %d, want %d", tt.transport, got, tt.priority) + } + }) + } + + // Verify priority ordering: TCP < WS < WSS < Unknown + if bzz.TransportTCP.Priority() >= bzz.TransportWS.Priority() { + t.Error("TCP priority should be lower (better) than WS") + } + if bzz.TransportWS.Priority() >= bzz.TransportWSS.Priority() { + t.Error("WS priority should be lower (better) than WSS") + } + if bzz.TransportWSS.Priority() >= bzz.TransportUnknown.Priority() { + t.Error("WSS priority should be lower (better) than Unknown") + } +} diff --git a/pkg/p2p/error.go b/pkg/p2p/error.go index 5ce54d12e10..4ec8e2cfea3 100644 --- a/pkg/p2p/error.go +++ b/pkg/p2p/error.go @@ -20,6 +20,8 @@ var ( ErrDialLightNode = errors.New("target peer is a light node") // ErrPeerBlocklisted is returned if peer is on blocklist ErrPeerBlocklisted = errors.New("peer blocklisted") + // ErrUnsupportedAddresses is returned when all peer addresses use unsupported transports + ErrUnsupportedAddresses = errors.New("no supported addresses") ) const ( diff --git a/pkg/p2p/libp2p/export_test.go b/pkg/p2p/libp2p/export_test.go index c85a85b54ae..7d7db8d5ee4 100644 --- a/pkg/p2p/libp2p/export_test.go +++ b/pkg/p2p/libp2p/export_test.go @@ -7,11 +7,13 @@ package libp2p import ( "context" + "github.com/ethersphere/bee/v2/pkg/bzz" handshake "github.com/ethersphere/bee/v2/pkg/p2p/libp2p/internal/handshake" libp2pm "github.com/libp2p/go-libp2p" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/network" libp2ppeer "github.com/libp2p/go-libp2p/core/peer" + ma "github.com/multiformats/go-multiaddr" ) func (s *Service) HandshakeService() *handshake.Service { @@ -56,3 +58,15 @@ func SetAutoTLSCertManager(o *Options, m autoTLSCertManager) { type AutoTLSCertManager = autoTLSCertManager var NewCompositeAddressResolver = newCompositeAddressResolver + +func (s *Service) FilterSupportedAddresses(addrs []ma.Multiaddr) []ma.Multiaddr { + return s.filterSupportedAddresses(addrs) +} + +func (s *Service) SetTransportFlags(hasTCP, hasWS, hasWSS bool) { + s.enabledTransports = map[bzz.TransportType]bool{ + bzz.TransportTCP: hasTCP, + bzz.TransportWS: hasWS, + bzz.TransportWSS: hasWSS, + } +} diff --git a/pkg/p2p/libp2p/filter_addresses_test.go b/pkg/p2p/libp2p/filter_addresses_test.go new file mode 100644 index 00000000000..a86b09e0b05 --- /dev/null +++ b/pkg/p2p/libp2p/filter_addresses_test.go @@ -0,0 +1,218 @@ +// Copyright 2020 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package libp2p_test + +import ( + "testing" + + ma "github.com/multiformats/go-multiaddr" +) + +func mustMultiaddr(t *testing.T, s string) ma.Multiaddr { + t.Helper() + addr, err := ma.NewMultiaddr(s) + if err != nil { + t.Fatalf("failed to create multiaddr from %q: %v", s, err) + } + return addr +} + +func TestFilterSupportedAddresses(t *testing.T) { + t.Parallel() + + // Plain TCP addresses (IPv4) + tcpPrivate := "/ip4/10.233.99.120/tcp/1634/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj" + tcpPublic := "/ip4/104.28.194.73/tcp/32002/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj" + tcpLoopback := "/ip4/127.0.0.1/tcp/1634/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj" + tcpIPv6Loopback := "/ip6/::1/tcp/1634/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj" + + // WSS addresses with TLS and SNI (full underlay format) + wssPrivate := "/ip4/10.233.99.120/tcp/1635/tls/sni/10-233-99-120.k2k4r8pr3m3aug5nudg2y039qfj2gxw6wnlx0e0ghzxufcn38soyp9z4.libp2p.direct/ws/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj" + wssPublic := "/ip4/104.28.194.73/tcp/32532/tls/sni/104-28-194-73.k2k4r8pr3m3aug5nudg2y039qfj2gxw6wnlx0e0ghzxufcn38soyp9z4.libp2p.direct/ws/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj" + wssLoopback := "/ip4/127.0.0.1/tcp/1635/tls/sni/127-0-0-1.k2k4r8pr3m3aug5nudg2y039qfj2gxw6wnlx0e0ghzxufcn38soyp9z4.libp2p.direct/ws/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj" + wssIPv6Loopback := "/ip6/::1/tcp/1635/tls/sni/0--1.k2k4r8pr3m3aug5nudg2y039qfj2gxw6wnlx0e0ghzxufcn38soyp9z4.libp2p.direct/ws/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj" + + // Plain WS address (no TLS) + wsPlain := "/ip4/127.0.0.1/tcp/1635/ws/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj" + + // All TCP addresses + allTCP := []string{tcpPrivate, tcpPublic, tcpLoopback, tcpIPv6Loopback} + // All WSS addresses + allWSS := []string{wssPrivate, wssPublic, wssLoopback, wssIPv6Loopback} + + tests := []struct { + name string + hasTCP bool + hasWS bool + hasWSS bool + inputAddrs []string + expectedCount int + }{ + { + name: "TCP only transport accepts all TCP addresses", + hasTCP: true, + hasWS: false, + hasWSS: false, + inputAddrs: allTCP, + expectedCount: 4, + }, + { + name: "TCP only transport rejects WSS addresses", + hasTCP: true, + hasWS: false, + hasWSS: false, + inputAddrs: allWSS, + expectedCount: 0, + }, + { + name: "WSS only transport accepts all WSS addresses", + hasTCP: false, + hasWS: false, + hasWSS: true, + inputAddrs: allWSS, + expectedCount: 4, + }, + { + name: "WSS only transport rejects TCP addresses", + hasTCP: false, + hasWS: false, + hasWSS: true, + inputAddrs: allTCP, + expectedCount: 0, + }, + { + name: "TCP and WSS transports accept mixed addresses", + hasTCP: true, + hasWS: false, + hasWSS: true, + inputAddrs: append(allTCP, allWSS...), + expectedCount: 8, + }, + { + name: "WS transport accepts plain WS but not WSS", + hasTCP: false, + hasWS: true, + hasWSS: false, + inputAddrs: []string{wsPlain, wssPublic}, + expectedCount: 1, + }, + { + name: "WSS transport does not accept plain WS", + hasTCP: false, + hasWS: false, + hasWSS: true, + inputAddrs: []string{wsPlain}, + expectedCount: 0, + }, + { + name: "No transports reject all addresses", + hasTCP: false, + hasWS: false, + hasWSS: false, + inputAddrs: append(allTCP, allWSS...), + expectedCount: 0, + }, + { + name: "Empty input returns empty output", + hasTCP: true, + hasWS: true, + hasWSS: true, + inputAddrs: []string{}, + expectedCount: 0, + }, + { + name: "Real node addresses with TCP and WSS", + hasTCP: true, + hasWS: false, + hasWSS: true, + inputAddrs: []string{tcpPrivate, wssPrivate, tcpPublic, wssPublic, tcpLoopback, wssLoopback, tcpIPv6Loopback, wssIPv6Loopback}, + expectedCount: 8, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + // Create a service for testing + s, _ := newService(t, 1, libp2pServiceOpts{}) + + // Set transport flags + s.SetTransportFlags(tc.hasTCP, tc.hasWS, tc.hasWSS) + + // Create multiaddrs from strings + addrs := make([]ma.Multiaddr, len(tc.inputAddrs)) + for i, addrStr := range tc.inputAddrs { + addrs[i] = mustMultiaddr(t, addrStr) + } + + // Filter addresses + filtered := s.FilterSupportedAddresses(addrs) + + // Verify count + if len(filtered) != tc.expectedCount { + t.Errorf("expected %d addresses, got %d", tc.expectedCount, len(filtered)) + } + }) + } +} + +func TestFilterSupportedAddresses_FullUnderlayAddresses(t *testing.T) { + t.Parallel() + + // Complete set of underlay addresses from a real full underlay node + fullAddrs := []string{ + "/ip4/10.233.99.120/tcp/1634/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj", + "/ip4/10.233.99.120/tcp/1635/tls/sni/10-233-99-120.k2k4r8pr3m3aug5nudg2y039qfj2gxw6wnlx0e0ghzxufcn38soyp9z4.libp2p.direct/ws/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj", + "/ip4/104.28.194.73/tcp/32002/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj", + "/ip4/104.28.194.73/tcp/32532/tls/sni/104-28-194-73.k2k4r8pr3m3aug5nudg2y039qfj2gxw6wnlx0e0ghzxufcn38soyp9z4.libp2p.direct/ws/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj", + "/ip4/127.0.0.1/tcp/1634/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj", + "/ip4/127.0.0.1/tcp/1635/tls/sni/127-0-0-1.k2k4r8pr3m3aug5nudg2y039qfj2gxw6wnlx0e0ghzxufcn38soyp9z4.libp2p.direct/ws/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj", + "/ip6/::1/tcp/1634/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj", + "/ip6/::1/tcp/1635/tls/sni/0--1.k2k4r8pr3m3aug5nudg2y039qfj2gxw6wnlx0e0ghzxufcn38soyp9z4.libp2p.direct/ws/p2p/QmfSx1ujzboapD5h2CiqTJqUy46FeTDwXBszB3XUCfKEEj", + } + + // Create multiaddrs + addrs := make([]ma.Multiaddr, len(fullAddrs)) + for i, addrStr := range fullAddrs { + addrs[i] = mustMultiaddr(t, addrStr) + } + + t.Run("TCP and WSS enabled accepts all full underlay addresses", func(t *testing.T) { + t.Parallel() + s, _ := newService(t, 1, libp2pServiceOpts{}) + s.SetTransportFlags(true, false, true) + + filtered := s.FilterSupportedAddresses(addrs) + // 4 TCP addresses + 4 WSS addresses = 8 + if len(filtered) != 8 { + t.Errorf("expected 8 addresses, got %d", len(filtered)) + } + }) + + t.Run("TCP only accepts half of full underlay addresses", func(t *testing.T) { + t.Parallel() + s, _ := newService(t, 1, libp2pServiceOpts{}) + s.SetTransportFlags(true, false, false) + + filtered := s.FilterSupportedAddresses(addrs) + // Only 4 TCP addresses + if len(filtered) != 4 { + t.Errorf("expected 4 addresses (TCP only), got %d", len(filtered)) + } + }) + + t.Run("WSS only accepts half of full underlay addresses", func(t *testing.T) { + t.Parallel() + s, _ := newService(t, 1, libp2pServiceOpts{}) + s.SetTransportFlags(false, false, true) + + filtered := s.FilterSupportedAddresses(addrs) + // Only 4 WSS addresses + if len(filtered) != 4 { + t.Errorf("expected 4 addresses (WSS only), got %d", len(filtered)) + } + }) +} diff --git a/pkg/p2p/libp2p/internal/reacher/reacher.go b/pkg/p2p/libp2p/internal/reacher/reacher.go index aa2ec9cf3e5..eac2cc016f9 100644 --- a/pkg/p2p/libp2p/internal/reacher/reacher.go +++ b/pkg/p2p/libp2p/internal/reacher/reacher.go @@ -77,7 +77,6 @@ func New(streamer p2p.Pinger, notifier p2p.ReachableNotifier, o *Options, log lo } func (r *reacher) manage() { - defer r.wg.Done() c := make(chan *peer) @@ -180,6 +179,10 @@ func (r *reacher) tryAcquirePeer() (*peer, time.Duration) { // Connected adds a new peer to the queue for testing reachability. func (r *reacher) Connected(overlay swarm.Address, addr ma.Multiaddr) { + if addr == nil { + return + } + r.mu.Lock() defer r.mu.Unlock() diff --git a/pkg/p2p/libp2p/libp2p.go b/pkg/p2p/libp2p/libp2p.go index 9765268754b..95587e17f19 100644 --- a/pkg/p2p/libp2p/libp2p.go +++ b/pkg/p2p/libp2p/libp2p.go @@ -121,9 +121,9 @@ type Service struct { networkStatus atomic.Int32 HeadersRWTimeout time.Duration autoNAT autonat.AutoNAT - enableWS bool autoTLSCertManager autoTLSCertManager zapLogger *zap.Logger + enabledTransports map[bzz.TransportType]bool } type lightnodes interface { @@ -543,9 +543,13 @@ func New(ctx context.Context, signer beecrypto.Signer, networkID uint64, overlay lightNodes: lightNodes, HeadersRWTimeout: o.HeadersRWTimeout, autoNAT: autoNAT, - enableWS: o.EnableWS, autoTLSCertManager: certManager, zapLogger: zapLogger, + enabledTransports: map[bzz.TransportType]bool{ + bzz.TransportTCP: true, // TCP transport is always included + bzz.TransportWS: o.EnableWS, + bzz.TransportWSS: o.EnableWSS, + }, } peerRegistry.setDisconnecter(s) @@ -789,7 +793,7 @@ func (s *Service) handleIncoming(stream network.Stream) { return } - s.notifyReacherConnected(overlay, peerID) + s.notifyReacherConnected(overlay, peerMultiaddrs) peerUserAgent := appendSpace(s.peerUserAgent(s.ctx, peerID)) s.networkStatus.Store(int32(p2p.NetworkStatusAvailable)) @@ -798,23 +802,44 @@ func (s *Service) handleIncoming(stream network.Stream) { s.logger.Debug("stream handler: successfully connected to peer (inbound)", "address", i.BzzAddress.Overlay, "light", i.LightString(), "user_agent", peerUserAgent) } -func (s *Service) notifyReacherConnected(overlay swarm.Address, peerID libp2ppeer.ID) { - if s.reacher == nil { - return +// isTransportSupported checks if the given transport type is supported by this service. +func (s *Service) isTransportSupported(t bzz.TransportType) bool { + return s.enabledTransports[t] +} + +// filterSupportedAddresses filters multiaddresses to only include those +// that are supported by the available transports (TCP, WS, WSS). +func (s *Service) filterSupportedAddresses(addrs []ma.Multiaddr) []ma.Multiaddr { + if len(addrs) == 0 { + return addrs } - peerAddrs := s.host.Peerstore().Addrs(peerID) - bestAddr := bzz.SelectBestAdvertisedAddress(peerAddrs, nil) + filtered := make([]ma.Multiaddr, 0, len(addrs)) + for _, addr := range addrs { + if s.isTransportSupported(bzz.ClassifyTransport(addr)) { + filtered = append(filtered, addr) + } + } - s.logger.Debug("selected reacher address", "peer_id", peerID, "selected_addr", bestAddr.String(), "advertised_count", len(peerAddrs)) + return filtered +} - underlay, err := buildFullMA(bestAddr, peerID) - if err != nil { - s.logger.Error(err, "stream handler: unable to build complete peer multiaddr", "peer", overlay, "multiaddr", bestAddr, "peer_id", peerID) - _ = s.Disconnect(overlay, "unable to build complete peer multiaddr") +func (s *Service) notifyReacherConnected(overlay swarm.Address, underlays []ma.Multiaddr) { + if s.reacher == nil { + return + } + + filteredAddrs := s.filterSupportedAddresses(underlays) + if len(filteredAddrs) == 0 { + s.logger.Debug("no supported addresses for reacher", "overlay", overlay, "total", len(underlays)) return } - s.reacher.Connected(overlay, underlay) + + bestAddr := bzz.SelectBestAdvertisedAddress(filteredAddrs, nil) + + s.logger.Debug("selected reacher address", "overlay", overlay, "address", bestAddr, "filtered", len(filteredAddrs), "total", len(underlays)) + + s.reacher.Connected(overlay, bestAddr) } func (s *Service) SetPickyNotifier(n p2p.PickyNotifier) { @@ -988,19 +1013,19 @@ func (s *Service) Connect(ctx context.Context, addrs []ma.Multiaddr) (address *b err = s.determineCurrentNetworkStatus(err) }() + filteredAddrs := s.filterSupportedAddresses(addrs) + if len(filteredAddrs) == 0 { + s.logger.Debug("no supported addresses to connect", "total_addrs", len(addrs)) + return nil, p2p.ErrUnsupportedAddresses + } + var info *libp2ppeer.AddrInfo var peerID libp2ppeer.ID var connectErr error skippedSelf := false // Try to connect to each underlay address one by one. - // - // TODO: investigate the issue when AddrInfo with multiple underlay - // addresses for the same peer is passed to the host.Host.Connect function - // and reachabiltiy Private is emitted on libp2p EventBus(), which results - // in weaker connectivity and failures in some integration tests. - - for _, addr := range addrs { + for _, addr := range filteredAddrs { // Extract the peer ID from the multiaddr. ai, err := libp2ppeer.AddrInfoFromP2pAddr(addr) if err != nil { @@ -1059,7 +1084,7 @@ func (s *Service) Connect(ctx context.Context, addrs []ma.Multiaddr) (address *b } if info == nil { - return nil, fmt.Errorf("unable to identify peer from addresses: %v", addrs) + return nil, fmt.Errorf("unable to identify peer from addresses: %v", filteredAddrs) } stream, err := s.newStreamForPeerID(ctx, info.ID, handshake.ProtocolName, handshake.ProtocolVersion, handshake.StreamName) @@ -1161,7 +1186,7 @@ func (s *Service) Connect(ctx context.Context, addrs []ma.Multiaddr) (address *b s.metrics.CreatedConnectionCount.Inc() - s.notifyReacherConnected(overlay, peerID) + s.notifyReacherConnected(overlay, peerMultiaddrs) peerUA := appendSpace(s.peerUserAgent(ctx, peerID)) loggerV1.Debug("successfully connected to peer (outbound)", "addresses", i.BzzAddress.ShortString(), "light", i.LightString(), "user_agent", peerUA) diff --git a/pkg/topology/kademlia/kademlia.go b/pkg/topology/kademlia/kademlia.go index d4b28e8eb1b..7cfabd1ff27 100644 --- a/pkg/topology/kademlia/kademlia.go +++ b/pkg/topology/kademlia/kademlia.go @@ -430,21 +430,24 @@ func (k *Kad) connectionAttemptsHandler(ctx context.Context, wg *sync.WaitGroup, switch err = k.connect(ctx, peer.addr, bzzAddr.Underlays); { case errors.Is(err, p2p.ErrNetworkUnavailable): - k.logger.Debug("network unavailable when reaching peer", "peer_overlay_address", peer.addr, "peer_underlay_addresses", bzzAddr.Underlays) + k.logger.Debug("network unavailable when reaching peer", "peer_address", peer.addr, "peer_underlays", bzzAddr.Underlays) return case errors.Is(err, errPruneEntry): - k.logger.Debug("dial to light node", "peer_overlay_address", peer.addr, "peer_underlay_addresses", bzzAddr.Underlays) + k.logger.Debug("dial to light node", "peer_address", peer.addr, "peer_underlays", bzzAddr.Underlays) remove(peer) return case errors.Is(err, errOverlayMismatch): - k.logger.Debug("overlay mismatch has occurred", "peer_overlay_address", peer.addr, "peer_underlay_addresses", bzzAddr.Underlays) + k.logger.Debug("overlay mismatch has occurred", "peer_address", peer.addr, "peer_underlays", bzzAddr.Underlays) remove(peer) return case errors.Is(err, p2p.ErrPeerBlocklisted): - k.logger.Debug("peer still in blocklist", "peer_address", bzzAddr) + k.logger.Debug("peer still in blocklist", "peer_address", peer.addr, "peer_underlays", bzzAddr.Underlays) + return + case errors.Is(err, p2p.ErrUnsupportedAddresses): + k.logger.Debug("peer has no supported addresses", "peer_address", peer.addr, "peer_underlays", bzzAddr.Underlays) return case err != nil: - k.logger.Debug("peer not reachable from kademlia", "peer_address", bzzAddr, "error", err) + k.logger.Debug("peer not reachable from kademlia", "peer_address", peer.addr, "peer_underlays", bzzAddr.Underlays, "error", err) return } @@ -981,6 +984,8 @@ func (k *Kad) connect(ctx context.Context, peer swarm.Address, ma []ma.Multiaddr return err case errors.Is(err, p2p.ErrPeerBlocklisted): return err + case errors.Is(err, p2p.ErrUnsupportedAddresses): + return err case err != nil: k.logger.Info("could not connect to peer", "peer_address", peer, "error", err) From db955eea800a4110ac154f1fdd8e0f973290caae Mon Sep 17 00:00:00 2001 From: acud <12988138+acud@users.noreply.github.com> Date: Fri, 13 Feb 2026 11:15:44 -0600 Subject: [PATCH 82/96] chore: remove timebomb (#5359) --- Makefile | 4 +--- cmd/bee/cmd/start.go | 3 --- cmd/bee/cmd/timebomb.go | 46 ----------------------------------------- 3 files changed, 1 insertion(+), 52 deletions(-) delete mode 100644 cmd/bee/cmd/timebomb.go diff --git a/Makefile b/Makefile index 047f5d4f8be..504f87cb190 100644 --- a/Makefile +++ b/Makefile @@ -19,11 +19,9 @@ BEE_API_VERSION ?= "$(shell grep '^ version:' openapi/Swarm.yaml | awk '{print VERSION ?= "$(shell git describe --tags --abbrev=0 | cut -c2-)" COMMIT_HASH ?= "$(shell git describe --long --dirty --always --match "" || true)" CLEAN_COMMIT ?= "$(shell git describe --long --always --match "" || true)" -COMMIT_TIME ?= "$(shell git show -s --format=%ct $(CLEAN_COMMIT) || true)" LDFLAGS ?= -s -w \ -X github.com/ethersphere/bee/v2.version="$(VERSION)" \ -X github.com/ethersphere/bee/v2.commitHash="$(COMMIT_HASH)" \ --X github.com/ethersphere/bee/v2.commitTime="$(COMMIT_TIME)" \ -X github.com/ethersphere/bee/v2/pkg/api.Version="$(BEE_API_VERSION)" \ -X github.com/ethersphere/bee/v2/pkg/p2p/libp2p.reachabilityOverridePublic="$(REACHABILITY_OVERRIDE_PUBLIC)" \ -X github.com/ethersphere/bee/v2/pkg/postage/listener.batchFactorOverridePublic="$(BATCHFACTOR_OVERRIDE_PUBLIC)" @@ -172,4 +170,4 @@ clean: $(GO) clean rm -rf dist/ -FORCE: \ No newline at end of file +FORCE: diff --git a/cmd/bee/cmd/start.go b/cmd/bee/cmd/start.go index 9633b9bd6f1..b279f893b4f 100644 --- a/cmd/bee/cmd/start.go +++ b/cmd/bee/cmd/start.go @@ -68,11 +68,8 @@ func (c *command) initStartCmd() (err error) { } fmt.Print(beeWelcomeMessage) - fmt.Printf("\n\nversion: %v - planned to be supported until %v, please follow https://ethswarm.org/\n\n", bee.Version, endSupportDate()) logger.Info("bee version", "version", bee.Version) - go startTimeBomb(logger) - // ctx is global context of bee node; which is canceled after interrupt signal is received. ctx, cancel := context.WithCancel(context.Background()) sysInterruptChannel := make(chan os.Signal, 1) diff --git a/cmd/bee/cmd/timebomb.go b/cmd/bee/cmd/timebomb.go deleted file mode 100644 index f53f74627c3..00000000000 --- a/cmd/bee/cmd/timebomb.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2021 The Swarm Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package cmd - -import ( - "strconv" - "time" - - "github.com/ethersphere/bee/v2" - "github.com/ethersphere/bee/v2/pkg/log" -) - -const ( - limitDays = 90 - warningDays = 0.9 * limitDays // show warning once 90% of the time bomb time has passed - sleepFor = 30 * time.Minute -) - -var ( - commitTime, _ = strconv.ParseInt(bee.CommitTime(), 10, 64) - versionReleased = time.Unix(commitTime, 0) -) - -func startTimeBomb(logger log.Logger) { - for { - outdated := time.Now().AddDate(0, 0, -limitDays) - - if versionReleased.Before(outdated) { - logger.Warning("your node is outdated, please check for the latest version") - } else { - almostOutdated := time.Now().AddDate(0, 0, -warningDays) - - if versionReleased.Before(almostOutdated) { - logger.Warning("your node is almost outdated, please check for the latest version") - } - } - - <-time.After(sleepFor) - } -} - -func endSupportDate() string { - return versionReleased.AddDate(0, 0, limitDays).Format("2 January 2006") -} From b2f67623a1342f61ccb450414e4140d4ce400a68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ljubi=C5=A1a=20Ga=C4=8Devi=C4=87?= <35105035+gacevicljubisa@users.noreply.github.com> Date: Fri, 13 Feb 2026 18:16:26 +0100 Subject: [PATCH 83/96] refactor(libp2p): extract p2p-forge code into separate file (#5349) --- pkg/p2p/libp2p/libp2p.go | 88 +++-------------------- pkg/p2p/libp2p/p2pforge.go | 138 +++++++++++++++++++++++++++++++++++++ 2 files changed, 149 insertions(+), 77 deletions(-) create mode 100644 pkg/p2p/libp2p/p2pforge.go diff --git a/pkg/p2p/libp2p/libp2p.go b/pkg/p2p/libp2p/libp2p.go index 95587e17f19..aae05dc45e0 100644 --- a/pkg/p2p/libp2p/libp2p.go +++ b/pkg/p2p/libp2p/libp2p.go @@ -7,13 +7,11 @@ package libp2p import ( "context" "crypto/ecdsa" - "crypto/tls" "errors" "fmt" "net" "net/netip" "os" - "path/filepath" "runtime" "slices" "strconv" @@ -22,7 +20,6 @@ import ( "time" ocprom "contrib.go.opencensus.io/exporter/prometheus" - "github.com/caddyserver/certmagic" "github.com/coreos/go-semver/semver" "github.com/ethersphere/bee/v2" "github.com/ethersphere/bee/v2/pkg/addressbook" @@ -41,6 +38,7 @@ import ( "github.com/ethersphere/bee/v2/pkg/topology" "github.com/ethersphere/bee/v2/pkg/topology/lightnode" "github.com/ethersphere/bee/v2/pkg/tracing" + p2pforge "github.com/ipshipyard/p2p-forge/client" "github.com/libp2p/go-libp2p" "github.com/libp2p/go-libp2p/config" "github.com/libp2p/go-libp2p/core/crypto" @@ -65,9 +63,6 @@ import ( "github.com/prometheus/client_golang/prometheus" "go.uber.org/atomic" "go.uber.org/zap" - "go.uber.org/zap/zapcore" - - p2pforge "github.com/ipshipyard/p2p-forge/client" ) // loggerName is the tree path name of the logger for this package. @@ -134,14 +129,6 @@ type lightnodes interface { EachPeer(pf topology.EachPeerFunc) error } -// autoTLSCertManager defines the interface for managing TLS certificates. -type autoTLSCertManager interface { - Start() error - Stop() - TLSConfig() *tls.Config - AddressFactory() config.AddrsFactory -} - type Options struct { PrivateKey *ecdsa.PrivateKey NATAddr string @@ -294,78 +281,25 @@ func New(ctx context.Context, signer beecrypto.Signer, networkID uint64, overlay if o.autoTLSCertManager != nil { certManager = o.autoTLSCertManager } else { - // create a zap logger needed for cert manager to be as close to - // swarm logger as possible - l, err := zap.Config{ - Level: zap.NewAtomicLevelAt(zap.DebugLevel), - Development: false, - Sampling: &zap.SamplingConfig{ - Initial: 100, - Thereafter: 100, - }, - Encoding: "json", - EncoderConfig: zapcore.EncoderConfig{ - TimeKey: "time", - LevelKey: "level", - NameKey: "logger", - CallerKey: "caller", - FunctionKey: zapcore.OmitKey, - MessageKey: "msg", - StacktraceKey: "stacktrace", - LineEnding: zapcore.DefaultLineEnding, - EncodeLevel: zapcore.LowercaseLevelEncoder, - EncodeTime: zapcore.EpochTimeEncoder, - EncodeDuration: zapcore.SecondsDurationEncoder, - EncodeCaller: zapcore.ShortCallerEncoder, - }, - OutputPaths: []string{"stderr"}, - ErrorOutputPaths: []string{"stderr"}, - }.Build() + forgeMgr, err := newP2PForgeCertManager(logger, P2PForgeOptions{ + Domain: o.AutoTLSDomain, + RegistrationEndpoint: o.AutoTLSRegistrationEndpoint, + CAEndpoint: o.AutoTLSCAEndpoint, + StorageDir: o.AutoTLSStorageDir, + }) if err != nil { return nil, err } - // assing zap logger as it needs to be synced when the service stops - zapLogger = l - - defer func() { - _ = zapLogger.Sync() - }() - - // Use AutoTLS storage dir with domain subdir for easier management - // of different registers. - storagePath := filepath.Join(o.AutoTLSStorageDir, o.AutoTLSDomain) - - if err := os.MkdirAll(storagePath, 0700); err != nil { - return nil, fmt.Errorf("create certificate storage directory %s: %w", storagePath, err) - } - - certManager, err = p2pforge.NewP2PForgeCertMgr( - p2pforge.WithForgeDomain(o.AutoTLSDomain), - p2pforge.WithForgeRegistrationEndpoint(o.AutoTLSRegistrationEndpoint), - p2pforge.WithCAEndpoint(o.AutoTLSCAEndpoint), - p2pforge.WithCertificateStorage(&certmagic.FileStorage{Path: storagePath}), - p2pforge.WithLogger(zapLogger.Sugar()), - p2pforge.WithUserAgent(userAgent()), - p2pforge.WithAllowPrivateForgeAddrs(), - p2pforge.WithRegistrationDelay(0), - p2pforge.WithOnCertLoaded(func() { - logger.Info("auto tls certificate is loaded") - }), - p2pforge.WithOnCertRenewed(func() { - logger.Info("auto tls certificate is renewed") - }), - ) - if err != nil { - return nil, fmt.Errorf("initialize AutoTLS: %w", err) - } + certManager = forgeMgr.CertMgr() + zapLogger = forgeMgr.ZapLogger() } defer func() { if returnErr != nil { - // certificate manager has to be stopped if service is not - // constructed + // call if service is not constructed certManager.Stop() + _ = zapLogger.Sync() } }() diff --git a/pkg/p2p/libp2p/p2pforge.go b/pkg/p2p/libp2p/p2pforge.go new file mode 100644 index 00000000000..fd1d13b22eb --- /dev/null +++ b/pkg/p2p/libp2p/p2pforge.go @@ -0,0 +1,138 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package libp2p + +import ( + "crypto/tls" + "fmt" + "os" + "path/filepath" + + "github.com/caddyserver/certmagic" + "github.com/ethersphere/bee/v2/pkg/log" + p2pforge "github.com/ipshipyard/p2p-forge/client" + "github.com/libp2p/go-libp2p/config" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +// P2PForgeOptions contains the configuration for creating a P2P Forge certificate manager. +type P2PForgeOptions struct { + Domain string + RegistrationEndpoint string + CAEndpoint string + StorageDir string +} + +// P2PForgeCertManager wraps the p2p-forge certificate manager with its associated zap logger. +type P2PForgeCertManager struct { + certMgr *p2pforge.P2PForgeCertMgr + zapLogger *zap.Logger +} + +// newP2PForgeCertManager creates a new P2P Forge certificate manager. +// It handles the creation of the storage directory and configures logging +// to match bee's verbosity level. +func newP2PForgeCertManager(beeLogger log.Logger, opts P2PForgeOptions) (*P2PForgeCertManager, error) { + zapLogger, err := newZapLogger(beeLogger) + if err != nil { + return nil, fmt.Errorf("create zap logger: %w", err) + } + + // Use storage dir with domain subdir for easier management of different registries. + storagePath := filepath.Join(opts.StorageDir, opts.Domain) + if err := os.MkdirAll(storagePath, 0o700); err != nil { + return nil, fmt.Errorf("create certificate storage directory %s: %w", storagePath, err) + } + + certMgr, err := p2pforge.NewP2PForgeCertMgr( + p2pforge.WithForgeDomain(opts.Domain), + p2pforge.WithForgeRegistrationEndpoint(opts.RegistrationEndpoint), + p2pforge.WithCAEndpoint(opts.CAEndpoint), + p2pforge.WithCertificateStorage(&certmagic.FileStorage{Path: storagePath}), + p2pforge.WithLogger(zapLogger.Sugar()), + p2pforge.WithUserAgent(userAgent()), + p2pforge.WithAllowPrivateForgeAddrs(), + p2pforge.WithRegistrationDelay(0), + p2pforge.WithOnCertLoaded(func() { + beeLogger.Info("auto tls certificate is loaded") + }), + p2pforge.WithOnCertRenewed(func() { + beeLogger.Info("auto tls certificate is renewed") + }), + ) + if err != nil { + return nil, fmt.Errorf("initialize P2P Forge: %w", err) + } + + return &P2PForgeCertManager{ + certMgr: certMgr, + zapLogger: zapLogger, + }, nil +} + +// CertMgr returns the underlying p2pforge.P2PForgeCertMgr. +func (m *P2PForgeCertManager) CertMgr() *p2pforge.P2PForgeCertMgr { + return m.certMgr +} + +// ZapLogger returns the zap logger used by the certificate manager. +func (m *P2PForgeCertManager) ZapLogger() *zap.Logger { + return m.zapLogger +} + +// autoTLSCertManager defines the interface for managing TLS certificates. +type autoTLSCertManager interface { + Start() error + Stop() + TLSConfig() *tls.Config + AddressFactory() config.AddrsFactory +} + +// newZapLogger creates a zap logger configured to match bee's verbosity level. +// This is used by third-party libraries (like p2p-forge) that require a zap logger. +func newZapLogger(beeLogger log.Logger) (*zap.Logger, error) { + cfg := zap.Config{ + Level: zap.NewAtomicLevelAt(beeVerbosityToZapLevel(beeLogger.Verbosity())), + Development: false, + Encoding: "json", + OutputPaths: []string{"stderr"}, + ErrorOutputPaths: []string{"stderr"}, + Sampling: &zap.SamplingConfig{ + Initial: 100, + Thereafter: 100, + }, + EncoderConfig: zapcore.EncoderConfig{ + TimeKey: "time", + LevelKey: "level", + NameKey: "logger", + CallerKey: "caller", + MessageKey: "msg", + StacktraceKey: "stacktrace", + LineEnding: zapcore.DefaultLineEnding, + EncodeLevel: zapcore.LowercaseLevelEncoder, + EncodeTime: zapcore.EpochTimeEncoder, + EncodeDuration: zapcore.SecondsDurationEncoder, + EncodeCaller: zapcore.ShortCallerEncoder, + }, + } + return cfg.Build() +} + +// beeVerbosityToZapLevel converts bee's log verbosity level to zap's log level. +func beeVerbosityToZapLevel(v log.Level) zapcore.Level { + switch { + case v <= log.VerbosityNone: + return zap.FatalLevel // effectively silences the logger + case v == log.VerbosityError: + return zap.ErrorLevel + case v == log.VerbosityWarning: + return zap.WarnLevel + case v == log.VerbosityInfo: + return zap.InfoLevel + default: + return zap.DebugLevel // VerbosityDebug and VerbosityAll + } +} From ffb7104855dcc1df1e30a1c7a55be575e342e955 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ljubi=C5=A1a=20Ga=C4=8Devi=C4=87?= <35105035+gacevicljubisa@users.noreply.github.com> Date: Fri, 13 Feb 2026 18:24:50 +0100 Subject: [PATCH 84/96] fix(p2p): update peer address on reconnect and optimize reacher (#5348) --- pkg/p2p/libp2p/internal/reacher/heap.go | 33 ++++ pkg/p2p/libp2p/internal/reacher/reacher.go | 81 +++++---- .../libp2p/internal/reacher/reacher_test.go | 162 ++++++++++++++++++ 3 files changed, 241 insertions(+), 35 deletions(-) create mode 100644 pkg/p2p/libp2p/internal/reacher/heap.go diff --git a/pkg/p2p/libp2p/internal/reacher/heap.go b/pkg/p2p/libp2p/internal/reacher/heap.go new file mode 100644 index 00000000000..10277b3df47 --- /dev/null +++ b/pkg/p2p/libp2p/internal/reacher/heap.go @@ -0,0 +1,33 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package reacher + +// peerHeap is a min-heap of peers ordered by retryAfter time. +type peerHeap []*peer + +func (h peerHeap) Len() int { return len(h) } +func (h peerHeap) Less(i, j int) bool { return h[i].retryAfter.Before(h[j].retryAfter) } +func (h peerHeap) Swap(i, j int) { + h[i], h[j] = h[j], h[i] + h[i].index = i + h[j].index = j +} + +func (h *peerHeap) Push(x any) { + n := len(*h) + p := x.(*peer) + p.index = n + *h = append(*h, p) +} + +func (h *peerHeap) Pop() any { + old := *h + n := len(old) + p := old[n-1] + old[n-1] = nil // avoid memory leak + p.index = -1 // for safety + *h = old[0 : n-1] + return p +} diff --git a/pkg/p2p/libp2p/internal/reacher/reacher.go b/pkg/p2p/libp2p/internal/reacher/reacher.go index eac2cc016f9..5bdc36c47f2 100644 --- a/pkg/p2p/libp2p/internal/reacher/reacher.go +++ b/pkg/p2p/libp2p/internal/reacher/reacher.go @@ -7,6 +7,7 @@ package reacher import ( + "container/heap" "context" "sync" "time" @@ -27,11 +28,13 @@ type peer struct { overlay swarm.Address addr ma.Multiaddr retryAfter time.Time + index int // index in the heap } type reacher struct { - mu sync.Mutex - peers map[string]*peer + mu sync.Mutex + peerHeap peerHeap // min-heap ordered by retryAfter + peerIndex map[string]*peer // lookup by overlay for O(1) access newPeer chan struct{} quit chan struct{} @@ -53,12 +56,13 @@ type Options struct { func New(streamer p2p.Pinger, notifier p2p.ReachableNotifier, o *Options, log log.Logger) *reacher { r := &reacher{ - newPeer: make(chan struct{}, 1), - quit: make(chan struct{}), - pinger: streamer, - peers: make(map[string]*peer), - notifier: notifier, - logger: log.WithName("reacher").Register(), + newPeer: make(chan struct{}, 1), + quit: make(chan struct{}), + pinger: streamer, + peerHeap: make(peerHeap, 0), + peerIndex: make(map[string]*peer), + notifier: notifier, + logger: log.WithName("reacher").Register(), } if o == nil { @@ -79,7 +83,7 @@ func New(streamer p2p.Pinger, notifier p2p.ReachableNotifier, o *Options, log lo func (r *reacher) manage() { defer r.wg.Done() - c := make(chan *peer) + c := make(chan peer) defer close(c) ctx, cancel := context.WithCancel(context.Background()) @@ -92,7 +96,7 @@ func (r *reacher) manage() { for { - p, tryAfter := r.tryAcquirePeer() + p, ok, tryAfter := r.tryAcquirePeer() // if no peer is returned, // wait until either more work or the closest retry-after time. @@ -110,7 +114,7 @@ func (r *reacher) manage() { } // wait for work - if p == nil { + if !ok { select { case <-r.quit: return @@ -128,7 +132,7 @@ func (r *reacher) manage() { } } -func (r *reacher) ping(c chan *peer, ctx context.Context) { +func (r *reacher) ping(c chan peer, ctx context.Context) { defer r.wg.Done() for p := range c { func() { @@ -146,38 +150,34 @@ func (r *reacher) ping(c chan *peer, ctx context.Context) { } } -func (r *reacher) tryAcquirePeer() (*peer, time.Duration) { +func (r *reacher) tryAcquirePeer() (peer, bool, time.Duration) { r.mu.Lock() defer r.mu.Unlock() - var ( - now = time.Now() - nextClosest time.Time - ) + if len(r.peerHeap) == 0 { + return peer{}, false, 0 + } - for _, p := range r.peers { + now := time.Now() - // retry after has expired, retry - if now.After(p.retryAfter) { - p.retryAfter = time.Now().Add(r.options.RetryAfterDuration) - return p, 0 - } + // Peek at the peer with the earliest retryAfter + p := r.peerHeap[0] - // here, we find the peer with the earliest retry after - if nextClosest.IsZero() || p.retryAfter.Before(nextClosest) { - nextClosest = p.retryAfter - } + // If retryAfter has not expired, return time to wait + if now.Before(p.retryAfter) { + return peer{}, false, time.Until(p.retryAfter) } - if nextClosest.IsZero() { - return nil, 0 - } + // Update retryAfter and fix heap position + p.retryAfter = time.Now().Add(r.options.RetryAfterDuration) + heap.Fix(&r.peerHeap, p.index) - // return the time to wait until the closest retry after - return nil, time.Until(nextClosest) + // Return a copy so callers can read fields without holding the lock. + return *p, true, 0 } // Connected adds a new peer to the queue for testing reachability. +// If the peer already exists, its address is updated. func (r *reacher) Connected(overlay swarm.Address, addr ma.Multiaddr) { if addr == nil { return @@ -186,8 +186,15 @@ func (r *reacher) Connected(overlay swarm.Address, addr ma.Multiaddr) { r.mu.Lock() defer r.mu.Unlock() - if _, ok := r.peers[overlay.ByteString()]; !ok { - r.peers[overlay.ByteString()] = &peer{overlay: overlay, addr: addr} + key := overlay.ByteString() + if existing, ok := r.peerIndex[key]; ok { + existing.addr = addr // Update address for reconnecting peer + existing.retryAfter = time.Time{} // Reset to trigger immediate re-ping + heap.Fix(&r.peerHeap, existing.index) + } else { + p := &peer{overlay: overlay, addr: addr} + r.peerIndex[key] = p + heap.Push(&r.peerHeap, p) } select { @@ -201,7 +208,11 @@ func (r *reacher) Disconnected(overlay swarm.Address) { r.mu.Lock() defer r.mu.Unlock() - delete(r.peers, overlay.ByteString()) + key := overlay.ByteString() + if p, ok := r.peerIndex[key]; ok { + heap.Remove(&r.peerHeap, p.index) + delete(r.peerIndex, key) + } } // Close stops the worker. Must be called once. diff --git a/pkg/p2p/libp2p/internal/reacher/reacher_test.go b/pkg/p2p/libp2p/internal/reacher/reacher_test.go index 515e05eab88..e86442ed402 100644 --- a/pkg/p2p/libp2p/internal/reacher/reacher_test.go +++ b/pkg/p2p/libp2p/internal/reacher/reacher_test.go @@ -7,6 +7,7 @@ package reacher_test import ( "context" "errors" + "sync" "testing" "time" @@ -127,6 +128,167 @@ func TestDisconnected(t *testing.T) { }) } +func TestAddressUpdateOnReconnect(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + // Use 1 worker and a known retry duration to make timing deterministic. + options := reacher.Options{ + PingTimeout: time.Second * 5, + Workers: 1, + RetryAfterDuration: time.Minute, + } + + overlay := swarm.RandAddress(t) + oldAddr, _ := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/7071/p2p/16Uiu2HAmTBuJT9LvNmBiQiNoTsxE5mtNy6YG3paw79m94CRa9sRb") + newAddr, _ := ma.NewMultiaddr("/ip4/192.168.1.1/tcp/7072/p2p/16Uiu2HAmTBuJT9LvNmBiQiNoTsxE5mtNy6YG3paw79m94CRa9sRb") + + var pingsMu sync.Mutex + var pings []ma.Multiaddr + pinged := make(chan struct{}, 8) + + pingFunc := func(_ context.Context, a ma.Multiaddr) (time.Duration, error) { + pingsMu.Lock() + pings = append(pings, a) + pingsMu.Unlock() + pinged <- struct{}{} + return 0, nil + } + + reachableFunc := func(addr swarm.Address, status p2p.ReachabilityStatus) {} + + mock := newMock(pingFunc, reachableFunc) + + r := reacher.New(mock, mock, &options, log.Noop) + testutil.CleanupCloser(t, r) + + // First connection with old address – triggers initial ping. + r.Connected(overlay, oldAddr) + + select { + case <-time.After(time.Second * 10): + t.Fatal("timed out waiting for initial ping") + case <-pinged: + } + + // Verify old address was pinged first. + pingsMu.Lock() + if len(pings) != 1 { + t.Fatalf("expected 1 ping after initial connect, got %d", len(pings)) + } + if !pings[0].Equal(oldAddr) { + t.Fatalf("first ping should use old address, got %s", pings[0]) + } + pingsMu.Unlock() + + // Reconnect with a new address — should trigger immediate re-ping. + r.Connected(overlay, newAddr) + + select { + case <-time.After(time.Second * 10): + t.Fatal("timed out waiting for reconnect ping") + case <-pinged: + } + + // Verify the reconnect pinged the new address. + pingsMu.Lock() + if len(pings) != 2 { + t.Fatalf("expected 2 pings after reconnect, got %d", len(pings)) + } + if !pings[1].Equal(newAddr) { + t.Fatalf("reconnect ping should use new address, got %s", pings[1]) + } + pingsMu.Unlock() + + // Advance time past the retry duration — should trigger a scheduled re-ping. + time.Sleep(options.RetryAfterDuration + time.Second) + + select { + case <-time.After(time.Second * 10): + t.Fatal("timed out waiting for scheduled re-ping") + case <-pinged: + } + + // Verify the scheduled re-ping used the new address. + pingsMu.Lock() + if len(pings) != 3 { + t.Fatalf("expected 3 pings after retry duration, got %d", len(pings)) + } + if !pings[2].Equal(newAddr) { + t.Fatalf("scheduled re-ping should use new address, got %s", pings[2]) + } + pingsMu.Unlock() + }) +} + +func TestHeapOrdering(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + // Use single worker to ensure sequential processing + options := reacher.Options{ + PingTimeout: time.Second * 5, + Workers: 1, + RetryAfterDuration: time.Second * 10, + } + + var pingOrder []swarm.Address + var pingOrderMu sync.Mutex + allPinged := make(chan struct{}) + + overlay1 := swarm.RandAddress(t) + overlay2 := swarm.RandAddress(t) + overlay3 := swarm.RandAddress(t) + addr, _ := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/7071/p2p/16Uiu2HAmTBuJT9LvNmBiQiNoTsxE5mtNy6YG3paw79m94CRa9sRb") + + pingFunc := func(_ context.Context, _ ma.Multiaddr) (time.Duration, error) { + return 0, nil + } + + reachableFunc := func(overlay swarm.Address, status p2p.ReachabilityStatus) { + pingOrderMu.Lock() + pingOrder = append(pingOrder, overlay) + if len(pingOrder) == 3 { + close(allPinged) + } + pingOrderMu.Unlock() + } + + mock := newMock(pingFunc, reachableFunc) + + r := reacher.New(mock, mock, &options, log.Noop) + testutil.CleanupCloser(t, r) + + // Add peers - they should all be pinged since retryAfter starts at zero + r.Connected(overlay1, addr) + r.Connected(overlay2, addr) + r.Connected(overlay3, addr) + + select { + case <-time.After(time.Second * 5): + t.Fatalf("test timed out, only %d peers pinged", len(pingOrder)) + case <-allPinged: + } + + // Verify all three peers were pinged + pingOrderMu.Lock() + defer pingOrderMu.Unlock() + + if len(pingOrder) != 3 { + t.Fatalf("expected 3 peers pinged, got %d", len(pingOrder)) + } + + // Verify all overlays are present (order may vary due to heap with same retryAfter) + seen := make(map[string]bool) + for _, o := range pingOrder { + seen[o.String()] = true + } + if !seen[overlay1.String()] || !seen[overlay2.String()] || !seen[overlay3.String()] { + t.Fatalf("not all peers were pinged") + } + }) +} + type mock struct { pingFunc func(context.Context, ma.Multiaddr) (time.Duration, error) reachableFunc func(swarm.Address, p2p.ReachabilityStatus) From 825174854d0e6585d584887b06da18788a706e10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ljubi=C5=A1a=20Ga=C4=8Devi=C4=87?= <35105035+gacevicljubisa@users.noreply.github.com> Date: Mon, 16 Feb 2026 13:49:27 +0100 Subject: [PATCH 85/96] fix(kademlia): set reachability status metric value to 1 instead of 0 (#5368) --- pkg/topology/kademlia/kademlia.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/topology/kademlia/kademlia.go b/pkg/topology/kademlia/kademlia.go index 7cfabd1ff27..6dae0c69fa5 100644 --- a/pkg/topology/kademlia/kademlia.go +++ b/pkg/topology/kademlia/kademlia.go @@ -279,7 +279,7 @@ func New( k.bgBroadcastCtx, k.bgBroadcastCancel = context.WithCancel(context.Background()) - k.metrics.ReachabilityStatus.WithLabelValues(p2p.ReachabilityStatusUnknown.String()).Set(0) + k.metrics.ReachabilityStatus.WithLabelValues(p2p.ReachabilityStatusUnknown.String()).Set(1) return k, nil } @@ -1363,7 +1363,8 @@ func (k *Kad) UpdateReachability(status p2p.ReachabilityStatus) { } k.logger.Debug("reachability updated", "reachability", status) k.reachability = status - k.metrics.ReachabilityStatus.WithLabelValues(status.String()).Set(0) + k.metrics.ReachabilityStatus.Reset() + k.metrics.ReachabilityStatus.WithLabelValues(status.String()).Set(1) } // UpdateReachability updates node reachability status. From 80d60867d5d49068f3092e9f5d42579d0407d9d1 Mon Sep 17 00:00:00 2001 From: acud <12988138+acud@users.noreply.github.com> Date: Mon, 16 Feb 2026 09:43:56 -0600 Subject: [PATCH 86/96] chore: improve openapi spec (#5363) --- openapi/Swarm.yaml | 202 +++++++++++++++++++-------------------- openapi/SwarmCommon.yaml | 40 ++++---- 2 files changed, 117 insertions(+), 125 deletions(-) diff --git a/openapi/Swarm.yaml b/openapi/Swarm.yaml index f03aa20a1c2..4d8be527b50 100644 --- a/openapi/Swarm.yaml +++ b/openapi/Swarm.yaml @@ -3,10 +3,10 @@ openapi: 3.0.3 info: version: 7.4.0 title: Bee API - description: "A list of the currently provided Interfaces to interact with the swarm, implementing file operations and sending messages" + description: "API endpoints for interacting with the Swarm network, supporting file operations, messaging, and node management" externalDocs: - description: Browse the documentation @ the Swarm Docs + description: Browse the documentation at the Swarm Docs url: "https://docs.ethswarm.org" servers: @@ -30,7 +30,7 @@ servers: paths: "/grantee": post: - summary: "Create grantee list" + summary: "Create a grantee list" tags: - ACT parameters: @@ -79,7 +79,7 @@ paths: "/grantee/{reference}": get: - summary: "Get grantee list" + summary: "Get the grantee list" tags: - ACT parameters: @@ -103,7 +103,7 @@ paths: "500": $ref: "SwarmCommon.yaml#/components/responses/500" patch: - summary: "Update grantee list" + summary: "Update the grantee list" description: "Add or remove grantees from an existing grantee list" tags: - ACT @@ -196,7 +196,7 @@ paths: "/bytes/{reference}": get: - summary: "Get referenced data" + summary: "Retrieve data by reference" tags: - Bytes parameters: @@ -227,7 +227,7 @@ paths: default: description: Default response head: - summary: Requests the headers containing the content type and length for the reference + summary: Retrieve headers containing the content type and length for the reference tags: - Bytes parameters: @@ -268,7 +268,7 @@ paths: "/chunks": post: - summary: "Upload chunk" + summary: "Upload a chunk" tags: - Chunk parameters: @@ -282,7 +282,7 @@ paths: - $ref: "SwarmCommon.yaml#/components/parameters/SwarmAct" - $ref: "SwarmCommon.yaml#/components/parameters/SwarmActHistoryAddress" requestBody: - description: Chunk binary data that has to have at least 8 bytes. + description: Chunk binary data containing at least 8 bytes. content: application/octet-stream: schema: @@ -293,7 +293,7 @@ paths: description: OK headers: "swarm-tag": - description: Tag UID if it was passed to the request `swarm-tag` header. + description: Tag UID from the request `swarm-tag` header if provided. schema: $ref: "SwarmCommon.yaml#/components/schemas/Uid" "swarm-act-history-address": @@ -313,8 +313,8 @@ paths: "/chunks/stream": get: - summary: "Upload stream of chunks" - description: "Returns a WebSocket connection on which stream of chunks can be uploaded. Each chunk sent is acknowledged using a binary response `0` which serves as confirmation of upload of single chunk. Chunks should be packaged as binary messages for uploading. If a tag is specified, the chunks will be streamed into local storage and then be uploaded to the network once the stream is closed. If a tag is not specified, the chunks will bypass local storage and be directly uploaded to the network through the stream as they arrive." + summary: "Stream chunks for upload" + description: "Establishes a WebSocket connection for streaming chunks. Each uploaded chunk receives a binary acknowledgment (`0`). Chunks are sent as binary messages. When a tag is specified, chunks are stored locally and uploaded to the network after the stream closes. Without a tag, chunks are directly uploaded to the network as they arrive." tags: - Chunk parameters: @@ -329,13 +329,9 @@ paths: description: Default response "/bzz": post: - summary: "Upload file or a collection of files" + summary: "Upload a file or collection of files" description: - "In order to upload a collection, user can send a multipart request with all the files populated in the form data with appropriate headers.\n\n - User can also upload a tar file along with the swarm-collection header. This will upload the tar file after extracting the entire directory structure.\n\n - If the swarm-collection header is absent, all requests (including tar files) are considered as single file uploads.\n\n - A multipart request is treated as a collection regardless of whether the swarm-collection header is present. This means in order to serve single files - uploaded as a multipart request, the swarm-index-document header should be used with the name of the file." + "Upload single files or collections of files. To upload a collection, send a multipart request with files in the form data with appropriate headers. Tar files can be uploaded with the `swarm-collection` header to extract and upload the directory structure. Without the `swarm-collection` header, requests are treated as single file uploads. Multipart requests are always treated as collections; use the `swarm-index-document` header to specify a single file to serve." tags: - BZZ parameters: @@ -400,7 +396,7 @@ paths: "/bzz/{reference}": get: - summary: "Get file or index document from a collection of files" + summary: "Retrieve a file or index document from a collection" tags: - BZZ parameters: @@ -442,7 +438,7 @@ paths: default: description: Default response head: - summary: Get the headers containing the content type and length for the reference + summary: Retrieve headers with content type and length for the reference tags: - BZZ parameters: @@ -467,7 +463,7 @@ paths: "/bzz/{reference}/{path}": get: - summary: "Get referenced file from a collection of files" + summary: "Retrieve a file from a collection by path" tags: - BZZ parameters: @@ -649,12 +645,12 @@ paths: required: true description: Swarm reference of the root hash post: - summary: Pin the root hash with the given reference + summary: Pin a root hash by reference tags: - Pinning responses: "200": - description: Pin already exists, so no operation + description: Pin already exists content: application/json: schema: @@ -674,12 +670,12 @@ paths: default: description: Default response delete: - summary: Unpin the root hash with the given reference + summary: Unpin a root hash by reference tags: - Pinning responses: "200": - description: Unpinning root hash with reference + description: Root hash has been unpinned content: application/json: schema: @@ -691,12 +687,12 @@ paths: default: description: Default response get: - summary: Get pinning status of the root hash with the given reference + summary: Get the pinning status of a root hash tags: - Pinning responses: "200": - description: Reference of the pinned root hash + description: The pinned root hash reference content: application/json: schema: @@ -729,7 +725,7 @@ paths: "/pins/check": get: - summary: Validate pinned chunks integerity + summary: Validate pinned chunks integrity tags: - Pinning parameters: @@ -738,7 +734,7 @@ paths: schema: $ref: "SwarmCommon.yaml#/components/schemas/SwarmOnlyReference" required: false - description: The number of items to skip before starting to collect the result set. + description: Optional reference to check; if not provided, all pinned references are checked responses: "200": description: List of checked root hash references @@ -753,7 +749,7 @@ paths: "/pss/send/{topic}/{targets}": post: - summary: Send to recipient or target with Postal Service for Swarm + summary: Send a message using the Postal Service for Swarm tags: - Postal Service for Swarm parameters: @@ -790,7 +786,7 @@ paths: "/pss/subscribe/{topic}": get: - summary: Subscribe for messages on the given topic. + summary: Subscribe to messages on a topic tags: - Postal Service for Swarm parameters: @@ -802,7 +798,7 @@ paths: description: Topic name responses: "200": - description: Returns a WebSocket with a subscription for incoming message data on the requested topic. + description: Establishes a WebSocket subscription for incoming messages on the topic "500": $ref: "SwarmCommon.yaml#/components/responses/500" default: @@ -822,7 +818,7 @@ paths: description: "Single Owner Chunk address (which may have multiple payloads)" responses: "200": - description: Returns a WebSocket with a subscription for incoming message data on the requested SOC address. + description: Establishes a WebSocket subscription for incoming messages on the Single Owner Chunk address "500": $ref: "SwarmCommon.yaml#/components/responses/500" default: @@ -830,7 +826,7 @@ paths: "/soc/{owner}/{id}": post: - summary: Upload single owner chunk + summary: Upload a Single Owner Chunk tags: - Single owner chunk parameters: @@ -839,13 +835,13 @@ paths: schema: $ref: "SwarmCommon.yaml#/components/schemas/EthereumAddress" required: true - description: Owner + description: Ethereum address of the chunk owner - in: path name: id schema: $ref: "SwarmCommon.yaml#/components/schemas/HexString" required: true - description: Id + description: Unique identifier for the chunk - in: query name: sig schema: @@ -862,7 +858,7 @@ paths: - $ref: "SwarmCommon.yaml#/components/parameters/SwarmActHistoryAddress" requestBody: required: true - description: The SOC binary data is composed of the span (8 bytes) and the at most 4KB payload. + description: The SOC binary data, composed of the span (8 bytes) and up to 4KB of payload. content: application/octet-stream: schema: @@ -889,7 +885,7 @@ paths: default: description: Default response get: - summary: Resolve Single Owner Chunk data + summary: Retrieve Single Owner Chunk data tags: - Single owner chunk parameters: @@ -904,7 +900,7 @@ paths: schema: $ref: "SwarmCommon.yaml#/components/schemas/HexString" required: true - description: Arbitrary identifier of the related data + description: Unique identifier for the chunk data - $ref: "SwarmCommon.yaml#/components/parameters/SwarmOnlyRootChunkParameter" - $ref: "SwarmCommon.yaml#/components/parameters/SwarmCache" - $ref: "SwarmCommon.yaml#/components/parameters/SwarmRedundancyStrategyParameter" @@ -932,7 +928,7 @@ paths: "/feeds/{owner}/{topic}": post: - summary: Create an initial feed root manifest + summary: Create a feed root manifest tags: - Feed parameters: @@ -941,13 +937,13 @@ paths: schema: $ref: "SwarmCommon.yaml#/components/schemas/EthereumAddress" required: true - description: Owner + description: Ethereum address of the feed owner - in: path name: topic schema: $ref: "SwarmCommon.yaml#/components/schemas/HexString" required: true - description: Topic + description: Topic identifier for the feed - in: query name: type schema: @@ -979,7 +975,7 @@ paths: default: description: Default response get: - summary: Find feed update + summary: Retrieve the latest feed update tags: - Feed parameters: @@ -988,13 +984,13 @@ paths: schema: $ref: "SwarmCommon.yaml#/components/schemas/EthereumAddress" required: true - description: Owner + description: Ethereum address of the feed owner - in: path name: topic schema: $ref: "SwarmCommon.yaml#/components/schemas/HexString" required: true - description: Topic + description: Topic identifier for the feed - in: query name: at schema: @@ -1047,7 +1043,7 @@ paths: "/stewardship/{reference}": get: - summary: "Check if content is available" + summary: "Check content availability" tags: - Stewardship parameters: @@ -1071,7 +1067,7 @@ paths: default: description: Default response put: - summary: "Re-upload content for specified root hash" + summary: "Re-upload content by reference" tags: - Stewardship parameters: @@ -1117,7 +1113,7 @@ paths: "/health": get: - summary: Get node overall health Status + summary: Get the overall health status of the node description: | Health Status will indicate node healthiness. @@ -1136,7 +1132,7 @@ paths: "/readiness": get: - summary: Readiness endpoint indicates if node is ready to start accepting traffic + summary: Check if the node is ready to accept traffic tags: - Status responses: @@ -1151,7 +1147,7 @@ paths: "/balances": get: - summary: Get the balances with all known peers including prepaid services + summary: Get balances with all known peers tags: - Balance responses: @@ -1168,7 +1164,7 @@ paths: "/balances/{address}": get: - summary: Get the balances with a specific peer including prepaid services + summary: Get the balance with a specific peer tags: - Balance parameters: @@ -1211,7 +1207,7 @@ paths: "/consumed": get: - summary: Get the past due consumption balances with all known peers + summary: Get past due consumption balances with all known peers tags: - Balance responses: @@ -1228,7 +1224,7 @@ paths: "/consumed/{address}": get: - summary: Get the past due consumption balance with a specific peer + summary: Get past due consumption balance with a specific peer tags: - Balance parameters: @@ -1254,7 +1250,7 @@ paths: "/chequebook/address": get: - summary: Get the address of the chequebook contract used + summary: Get the chequebook contract address tags: - Chequebook responses: @@ -1284,7 +1280,7 @@ paths: "/chunks/{address}": get: - summary: "Get chunk" + summary: "Retrieve a chunk" tags: - Chunk parameters: @@ -1319,7 +1315,7 @@ paths: default: description: Default response head: - summary: Check if chunk at address exists locally + summary: Check if a chunk exists locally tags: - Chunk parameters: @@ -1344,7 +1340,7 @@ paths: "/envelope/{address}": post: - summary: "Create postage stamp signature against given chunk address" + summary: "Create a postage stamp for a chunk" tags: - Envelope parameters: @@ -1371,7 +1367,7 @@ paths: "/connect/{multiAddress}": post: - summary: Connect to address + summary: Connect to a peer address tags: - Connectivity parameters: @@ -1398,7 +1394,7 @@ paths: "/reservestate": get: - summary: Get reserve state + summary: Get the reserve state tags: - Status responses: @@ -1413,7 +1409,7 @@ paths: "/chainstate": get: - summary: Get chain state + summary: Get the chain state tags: - Status responses: @@ -1428,7 +1424,7 @@ paths: "/node": get: - summary: Get information about the node + summary: Get node information tags: - Status responses: @@ -1443,7 +1439,7 @@ paths: "/peers": get: - summary: Get a list of peers + summary: Get the list of connected peers tags: - Connectivity responses: @@ -1458,7 +1454,7 @@ paths: "/peers/{address}": delete: - summary: Remove peer + summary: Disconnect from a peer tags: - Connectivity parameters: @@ -1470,7 +1466,7 @@ paths: description: Swarm address of peer responses: "200": - description: Disconnected peer + description: Peer has been disconnected content: application/json: schema: @@ -1484,7 +1480,7 @@ paths: "/pingpong/{address}": post: - summary: Try connection to node + summary: Ping a peer to measure latency tags: - Connectivity parameters: @@ -1512,7 +1508,7 @@ paths: "/settlements/{address}": get: - summary: Get amount of sent and received from settlements with a peer + summary: Get settlement amounts sent and received with a peer tags: - Settlements parameters: @@ -1524,7 +1520,7 @@ paths: description: Swarm address of peer responses: "200": - description: Amount of sent or received from settlements with a peer + description: Settlement amounts sent and received with the peer content: application/json: schema: @@ -1538,7 +1534,7 @@ paths: "/settlements": get: - summary: Get settlements with all known peers and total amount sent or received + summary: Get settlements with all known peers and totals tags: - Settlements responses: @@ -1555,7 +1551,7 @@ paths: "/timesettlements": get: - summary: Get time based settlements with all known peers and total amount sent or received + summary: Get time-based settlements with all known peers and totals tags: - Settlements responses: @@ -1572,7 +1568,7 @@ paths: "/topology": get: - summary: Get topology of known network + summary: Get the network topology tags: - Connectivity responses: @@ -1585,7 +1581,7 @@ paths: "/welcome-message": get: - summary: Get configured P2P welcome message + summary: Get the P2P welcome message tags: - Connectivity responses: @@ -1600,7 +1596,7 @@ paths: default: description: Default response post: - summary: Set P2P welcome message + summary: Set the P2P welcome message tags: - Connectivity requestBody: @@ -1624,7 +1620,7 @@ paths: "/chequebook/cashout/{peer-id}": get: - summary: Get last cashout action for the peer + summary: Get the last cashout status for a peer parameters: - in: path name: peer-id @@ -1648,7 +1644,7 @@ paths: default: description: Default response post: - summary: Cashout the last cheque for the peer + summary: Cash out the last cheque for a peer parameters: - in: path name: peer-id @@ -1662,7 +1658,7 @@ paths: - Chequebook responses: "201": - description: OK + description: Cheque has been cashed out content: application/json: schema: @@ -1678,7 +1674,7 @@ paths: "/chequebook/cheque/{peer-id}": get: - summary: Get last cheques for the peer + summary: Get the last cheques for a peer parameters: - in: path name: peer-id @@ -1690,7 +1686,7 @@ paths: - Chequebook responses: "200": - description: Last cheques + description: The last cheques for the peer content: application/json: schema: @@ -1704,12 +1700,12 @@ paths: "/chequebook/cheque": get: - summary: Get last cheques for all peers + summary: Get the last cheques for all peers tags: - Chequebook responses: "200": - description: Last cheques + description: The last cheques for all peers content: application/json: schema: @@ -1723,14 +1719,14 @@ paths: "/chequebook/deposit": post: - summary: Deposit tokens from overlay address into chequebook + summary: Deposit tokens into the chequebook parameters: - in: query name: amount schema: type: integer required: true - description: amount of tokens to deposit + description: Amount of tokens to deposit - $ref: "SwarmCommon.yaml#/components/parameters/GasPriceParameter" tags: - Chequebook @@ -1750,14 +1746,14 @@ paths: "/chequebook/withdraw": post: - summary: Withdraw tokens from the chequebook to the overlay address + summary: Withdraw tokens from the chequebook parameters: - in: query name: amount schema: type: integer required: true - description: amount of tokens to withdraw + description: Amount of tokens to withdraw - $ref: "SwarmCommon.yaml#/components/parameters/GasPriceParameter" tags: - Chequebook @@ -1794,7 +1790,7 @@ paths: "/transactions/{txHash}": get: - summary: Get information about a sent transaction + summary: Retrieve transaction information parameters: - in: path name: txHash @@ -1806,7 +1802,7 @@ paths: - Transaction responses: "200": - description: Get info about transaction + description: Transaction information content: application/json: schema: @@ -1818,7 +1814,7 @@ paths: default: description: Default response post: - summary: Rebroadcast existing transaction + summary: Rebroadcast a transaction parameters: - in: path name: txHash @@ -1869,12 +1865,12 @@ paths: "/stamps": get: - summary: Get stamps for this node + summary: Get postage stamps for this node tags: - Postage Stamps responses: "200": - description: Returns an array of postage batches. + description: An array of postage stamps content: application/json: schema: @@ -1941,7 +1937,7 @@ paths: post: summary: Buy a new postage batch. description: | - Be aware, this endpoint creates an on-chain transactions and transfers BZZ from the node's Ethereum account and hence directly manipulates the wallet balance! + Be aware, this endpoint creates an on-chain transaction and transfers BZZ from the node's Ethereum account, directly affecting the wallet balance! tags: - Postage Stamps parameters: @@ -1956,7 +1952,7 @@ paths: schema: type: integer required: true - description: Batch depth which specifies how many chunks can be signed with the batch. It is a logarithm. Must be higher than default bucket depth (16) + description: Batch depth (logarithm) specifying the maximum number of chunks this stamp can cover. Must be greater than the default bucket depth (16) - in: query name: label schema: @@ -1990,7 +1986,7 @@ paths: patch: summary: Top up an existing postage batch. description: | - Be aware, this endpoint creates on-chain transactions and transfers BZZ from the node's Ethereum account and hence directly manipulates the wallet balance! + Be aware, this endpoint creates an on-chain transaction and transfers BZZ from the node's Ethereum account, directly affecting the wallet balance! tags: - Postage Stamps parameters: @@ -2030,7 +2026,7 @@ paths: patch: summary: Dilute an existing postage batch. description: | - Be aware, this endpoint creates on-chain transactions and transfers BZZ from the node's Ethereum account and hence directly manipulates the wallet balance! + Be aware, this endpoint creates an on-chain transaction and transfers BZZ from the node's Ethereum account, directly affecting the wallet balance! tags: - Postage Stamps parameters: @@ -2045,7 +2041,7 @@ paths: schema: type: integer required: true - description: New batch depth. Must be higher than the previous depth. + description: The new batch depth, which must be greater than the current depth - $ref: "SwarmCommon.yaml#/components/parameters/GasPriceParameter" - $ref: "SwarmCommon.yaml#/components/parameters/GasLimitParameter" responses: @@ -2066,12 +2062,12 @@ paths: "/batches": get: - summary: Get all globally available batches that were purchased by all nodes. + summary: Get all globally available postage batches tags: - Postage Stamps responses: "200": - description: Returns an array of all available and currently valid postage batches. + description: An array of all available and valid postage batches content: application/json: schema: @@ -2120,7 +2116,7 @@ paths: "/accounting": get: - summary: Get all accounting associated values with all known peers + summary: Get accounting values for all known peers tags: - Balance responses: @@ -2137,7 +2133,7 @@ paths: "/redistributionstate": get: - summary: Get current status of node in redistribution game + summary: Get the node's redistribution game status tags: - RedistributionState responses: @@ -2155,7 +2151,7 @@ paths: description: Default response "/wallet": get: - summary: Get wallet balance for BZZ and xDai + summary: Get wallet balance for BZZ and xDAI tags: - Wallet responses: @@ -2171,7 +2167,7 @@ paths: description: Default response "/wallet/withdraw/{coin}": post: - summary: Allows withdrawals of BZZ or xDAI to provided (whitelisted) address + summary: Withdraw BZZ or xDAI to a whitelisted address tags: - Wallet parameters: @@ -2246,8 +2242,8 @@ paths: "/stake/{amount}": post: - summary: Deposit some amount for staking. - description: Be aware, this endpoint creates an on-chain transactions and transfers BZZ from the node's Ethereum account and hence directly manipulates the wallet balance. + summary: Deposit an amount for staking. + description: Be aware, this endpoint creates an on-chain transaction and transfers BZZ from the node's Ethereum account, directly affecting the wallet balance. tags: - Staking parameters: @@ -2291,8 +2287,8 @@ paths: default: description: Default response delete: - summary: Withdraws all past staked amount back to the wallet. - description: Be aware, this endpoint can only be called when the contract is paused and is in the process of being migrated to a new contract. + summary: Withdraw all previously staked amounts. + description: Be aware, this endpoint can only be called when the contract is paused and undergoing migration to a new contract. tags: - Staking parameters: diff --git a/openapi/SwarmCommon.yaml b/openapi/SwarmCommon.yaml index 1c96d34bf7d..ab39fa48555 100644 --- a/openapi/SwarmCommon.yaml +++ b/openapi/SwarmCommon.yaml @@ -2,8 +2,7 @@ openapi: 3.0.3 info: version: 4.3.0 title: Common Data Types - description: | - \*****bzzz***** + description: Common data structures and types used throughout the Bee API externalDocs: description: Browse the documentation @ the Swarm Docs @@ -37,7 +36,7 @@ components: $ref: "#/components/schemas/PublicKey" BigInt: - description: Numeric string that represents integer which might exceed `Number.MAX_SAFE_INTEGER` limit (2^53-1) + description: Numeric string representing an integer that may exceed `Number.MAX_SAFE_INTEGER` (2^53-1) type: string example: "1000000000000000000" @@ -242,12 +241,12 @@ components: example: "2020-06-11T11:26:42.6969797+02:00" Duration: - description: Go time.Duration format + description: Time duration in Go time.Duration format (e.g., 5.0018ms) type: string example: "5.0018ms" Seconds: - description: Go time.Duration format in seconds + description: Time duration in seconds (Go time.Duration format) type: number example: 30.5 @@ -507,7 +506,7 @@ components: utilization: type: integer usable: - description: Indicate that the batch was discovered by the Bee node, but it awaits enough on-chain confirmations before declaring the batch as usable. + description: Indicates whether the batch was discovered by the Bee node and has received sufficient on-chain confirmations type: boolean label: type: string @@ -834,7 +833,7 @@ components: LoggerExp: type: string - description: Base 64 encoded regular expression or subsystem string. + description: Base64-encoded regular expression or subsystem string pattern: "^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$" example: "b25lL25hbWU=" @@ -1110,7 +1109,7 @@ components: type: boolean required: false description: > - Represents if the uploaded data should be also locally pinned on the node. + Indicates whether the uploaded data should also be locally pinned on this node SwarmEncryptParameter: in: header @@ -1119,7 +1118,7 @@ components: type: boolean required: false description: > - Represents the encrypting state of the file + Indicates whether the file should be encrypted SwarmRedundancyLevelParameter: in: header @@ -1140,14 +1139,11 @@ components: enum: [0, 1, 2, 3] required: false description: > - Specify the retrieve strategy on redundant data. - The numbers stand for NONE, DATA, PROX and RACE, respectively. - Strategy NONE means no prefetching takes place. - Strategy DATA means only data chunks are prefetched. - Strategy PROX means only chunks that are close to the node are prefetched. - Strategy RACE means all chunks are prefetched: n data chunks and k parity chunks. The first n chunks to arrive are used to reconstruct the file. - Multiple strategies can be used in a fallback cascade if the swarm redundancy fallback mode is set to true. - The default strategy is NONE, DATA, falling back to PROX, falling back to RACE + Specify the retrieval strategy for redundant data. + Values represent: NONE (0), DATA (1), PROX (2), RACE (3). + NONE: no prefetching. DATA: prefetch only data chunks. PROX: prefetch chunks near this node. RACE: prefetch all chunks and use the first n to arrive. + Multiple strategies can be cascaded if fallback mode is enabled. + Default: NONE > DATA > PROX > RACE SwarmRedundancyFallbackModeParameter: in: header @@ -1189,7 +1185,7 @@ components: type: string example: index.html required: false - description: Default file to be referenced on path, if exists under that path + description: Default file to serve when a directory path is accessed SwarmErrorDocumentParameter: in: header @@ -1198,7 +1194,7 @@ components: type: string example: error.html required: false - description: Configure custom error document to be returned when a specified path can not be found in collection + description: Custom error document to return when a path is not found in the collection SwarmCollection: in: header @@ -1238,7 +1234,7 @@ components: default: "true" required: false description: > - Determines if the uploaded data should be sent to the network immediately or in a deferred fashion. By default the upload will be deferred. + Indicates whether the uploaded data should be sent to the network immediately or deferred. Default: deferred (true) SwarmCache: in: header @@ -1247,7 +1243,7 @@ components: type: boolean default: "true" required: false - description: "Determines if the download data should be cached on the node. By default the download will be cached" + description: "Indicates whether downloaded data should be cached on the node. Default: cached (true)" SwarmAct: in: header @@ -1285,7 +1281,7 @@ components: responses: "200": - description: OK. + description: Success "204": description: The resource was deleted successfully. "400": From 07cc20dba004153f238099b5041241579bd5c59e Mon Sep 17 00:00:00 2001 From: Calin M Date: Tue, 17 Feb 2026 15:03:04 +0200 Subject: [PATCH 87/96] chore: remove unused code and enable extra linters (#5369) --- .golangci.yml | 1 + pkg/manifest/mantaray/marshal.go | 5 ----- pkg/manifest/mantaray/node.go | 15 --------------- pkg/pusher/pusher.go | 3 +-- 4 files changed, 2 insertions(+), 22 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 776f2575cbe..f6c9c2e8bed 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -23,6 +23,7 @@ linters: - promlinter - thelper - unconvert + - unused settings: goheader: values: diff --git a/pkg/manifest/mantaray/marshal.go b/pkg/manifest/mantaray/marshal.go index 6d355721c1c..d95282135ad 100644 --- a/pkg/manifest/mantaray/marshal.go +++ b/pkg/manifest/mantaray/marshal.go @@ -190,11 +190,6 @@ func (bb *bitsForBytes) set(b byte) { bb.bits[b/8] |= 1 << (b % 8) } -//nolint:unused -func (bb *bitsForBytes) get(b byte) bool { - return bb.getUint8(b) -} - func (bb *bitsForBytes) getUint8(i uint8) bool { return (bb.bits[i/8]>>(i%8))&1 > 0 } diff --git a/pkg/manifest/mantaray/node.go b/pkg/manifest/mantaray/node.go index 88d4a2cbc6a..865a89a29eb 100644 --- a/pkg/manifest/mantaray/node.go +++ b/pkg/manifest/mantaray/node.go @@ -105,25 +105,10 @@ func (n *Node) makeWithMetadata() { n.nodeType = n.nodeType | nodeTypeWithMetadata } -//nolint:unused -func (n *Node) makeNotValue() { - n.nodeType = (nodeTypeMask ^ nodeTypeValue) & n.nodeType -} - -//nolint:unused -func (n *Node) makeNotEdge() { - n.nodeType = (nodeTypeMask ^ nodeTypeEdge) & n.nodeType -} - func (n *Node) makeNotWithPathSeparator() { n.nodeType = (nodeTypeMask ^ nodeTypeWithPathSeparator) & n.nodeType } -//nolint:unused -func (n *Node) makeNotWithMetadata() { - n.nodeType = (nodeTypeMask ^ nodeTypeWithMetadata) & n.nodeType -} - func (n *Node) SetObfuscationKey(obfuscationKey []byte) { bytes := make([]byte, 32) copy(bytes, obfuscationKey) diff --git a/pkg/pusher/pusher.go b/pkg/pusher/pusher.go index 46632c18765..9c4073f1fcc 100644 --- a/pkg/pusher/pusher.go +++ b/pkg/pusher/pusher.go @@ -63,8 +63,7 @@ type Service struct { } const ( - traceDuration = 30 * time.Second // duration for every root tracing span - ConcurrentPushes = swarm.Branches // how many chunks to push simultaneously + ConcurrentPushes = swarm.Branches // how many chunks to push simultaneously DefaultRetryCount = 6 ) From 48c19eff51d33000007fd921af0b2a651c7493c8 Mon Sep 17 00:00:00 2001 From: Calin M Date: Tue, 17 Feb 2026 15:03:48 +0200 Subject: [PATCH 88/96] feat: optimize postage batch snapshot (#5343) --- pkg/node/snapshot.go | 4 ++ pkg/node/snapshot_test.go | 82 ++++++++++++++++++++++++ pkg/postage/batchservice/batchservice.go | 42 +++++++++--- pkg/postage/listener/listener.go | 14 +++- 4 files changed, 130 insertions(+), 12 deletions(-) diff --git a/pkg/node/snapshot.go b/pkg/node/snapshot.go index aac0a6eb140..ef190c0e26c 100644 --- a/pkg/node/snapshot.go +++ b/pkg/node/snapshot.go @@ -50,6 +50,10 @@ func NewSnapshotLogFilterer(logger log.Logger, getter SnapshotGetter) *SnapshotL } } +func (f *SnapshotLogFilterer) GetBatchSnapshot() []byte { + return f.getter.GetBatchSnapshot() +} + // loadSnapshot is responsible for loading and processing the snapshot data. // It is intended to be called exactly once by initOnce.Do. func (f *SnapshotLogFilterer) loadSnapshot() error { diff --git a/pkg/node/snapshot_test.go b/pkg/node/snapshot_test.go index 81edbcaad5f..eab031067fd 100644 --- a/pkg/node/snapshot_test.go +++ b/pkg/node/snapshot_test.go @@ -20,6 +20,8 @@ import ( "github.com/ethersphere/bee/v2/pkg/postage/listener" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + archive "github.com/ethersphere/batch-archive" ) type mockSnapshotGetter struct { @@ -33,6 +35,12 @@ func (m mockSnapshotGetter) GetBatchSnapshot() []byte { return m.data } +type realSnapshotGetter struct{} + +func (r realSnapshotGetter) GetBatchSnapshot() []byte { + return archive.GetBatchSnapshot() +} + func makeSnapshotData(logs []types.Log) []byte { var buf bytes.Buffer gz := gzip.NewWriter(&buf) @@ -149,3 +157,77 @@ func TestNewSnapshotLogFilterer(t *testing.T) { assert.Equal(t, 0, res[3].Topics[0].Cmp(common.HexToHash("0xa4"))) }) } + +func TestSnapshotLogFilterer_RealSnapshot(t *testing.T) { + t.Parallel() + + getter := realSnapshotGetter{} + filterer := node.NewSnapshotLogFilterer(log.Noop, getter) + + t.Run("block number", func(t *testing.T) { + blockNumber, err := filterer.BlockNumber(context.Background()) + assert.NoError(t, err) + assert.Greater(t, blockNumber, uint64(0)) + }) + + t.Run("filter range", func(t *testing.T) { + // arbitrary range that should exist in the snapshot + from := big.NewInt(20000000) + to := big.NewInt(20001000) + res, err := filterer.FilterLogs(context.Background(), ethereum.FilterQuery{ + FromBlock: from, + ToBlock: to, + }) + require.NoError(t, err) + for _, l := range res { + assert.GreaterOrEqual(t, l.BlockNumber, from.Uint64()) + assert.LessOrEqual(t, l.BlockNumber, to.Uint64()) + } + }) + + t.Run("filter address mismatch", func(t *testing.T) { + // random address that should not match the postage stamp contract + addr := common.HexToAddress("0x1234567890123456789012345678901234567890") + res, err := filterer.FilterLogs(context.Background(), ethereum.FilterQuery{ + Addresses: []common.Address{addr}, + }) + require.NoError(t, err) + assert.Empty(t, res) + }) +} + +func BenchmarkNewSnapshotLogFilterer_Load(b *testing.B) { + getter := realSnapshotGetter{} + b.ResetTimer() + for i := 0; i < b.N; i++ { + filterer := node.NewSnapshotLogFilterer(log.Noop, getter) + _, err := filterer.BlockNumber(context.Background()) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkSnapshotLogFilterer(b *testing.B) { + getter := realSnapshotGetter{} + filterer := node.NewSnapshotLogFilterer(log.Noop, getter) + // ensure loaded + if _, err := filterer.BlockNumber(context.Background()); err != nil { + b.Fatal(err) + } + + b.Run("FilterLogs", func(b *testing.B) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + from := big.NewInt(20000000) + to := big.NewInt(20001000) + _, err := filterer.FilterLogs(context.Background(), ethereum.FilterQuery{ + FromBlock: from, + ToBlock: to, + }) + if err != nil { + b.Fatal(err) + } + } + }) +} diff --git a/pkg/postage/batchservice/batchservice.go b/pkg/postage/batchservice/batchservice.go index 6ad76b0bed4..aac224c1a89 100644 --- a/pkg/postage/batchservice/batchservice.go +++ b/pkg/postage/batchservice/batchservice.go @@ -40,6 +40,8 @@ type batchService struct { checksum hash.Hash // checksum hasher resync bool + + pendingChainState *postage.ChainState } type Interface interface { @@ -95,15 +97,22 @@ func New( } } - return &batchService{stateStore, storer, logger.WithName(loggerName).Register(), listener, owner, batchListener, sum, resync}, nil + return &batchService{stateStore: stateStore, storer: storer, logger: logger.WithName(loggerName).Register(), listener: listener, owner: owner, batchListener: batchListener, checksum: sum, resync: resync}, nil +} + +func (svc *batchService) getChainState() *postage.ChainState { + if svc.pendingChainState != nil { + return svc.pendingChainState + } + return svc.storer.GetChainState() } // Create will create a new batch with the given ID, owner value and depth and // stores it in the BatchedStore. func (svc *batchService) Create(id, owner []byte, totalAmout, normalisedBalance *big.Int, depth, bucketDepth uint8, immutable bool, txHash common.Hash) error { - // don't add batches which have value which equals total cumulative + // dont add batches which have value which equals total cumulative // payout or that are going to expire already within the next couple of blocks - val := big.NewInt(0).Add(svc.storer.GetChainState().TotalAmount, svc.storer.GetChainState().CurrentPrice) + val := big.NewInt(0).Add(svc.getChainState().TotalAmount, svc.getChainState().CurrentPrice) if normalisedBalance.Cmp(val) <= 0 { // don't do anything return fmt.Errorf("batch service: batch %x: %w", id, ErrZeroValueBatch) @@ -112,7 +121,7 @@ func (svc *batchService) Create(id, owner []byte, totalAmout, normalisedBalance ID: id, Owner: owner, Value: normalisedBalance, - Start: svc.storer.GetChainState().Block, + Start: svc.getChainState().Block, Depth: depth, BucketDepth: bucketDepth, Immutable: immutable, @@ -196,10 +205,13 @@ func (svc *batchService) UpdateDepth(id []byte, depth uint8, normalisedBalance * // UpdatePrice implements the EventUpdater interface. It sets the current // price from the chain in the service chain state. func (svc *batchService) UpdatePrice(price *big.Int, txHash common.Hash) error { - cs := svc.storer.GetChainState() + cs := svc.getChainState() cs.CurrentPrice = price - if err := svc.storer.PutChainState(cs); err != nil { - return fmt.Errorf("put chain state: %w", err) + + if svc.pendingChainState == nil { + if err := svc.storer.PutChainState(cs); err != nil { + return fmt.Errorf("put chain state: %w", err) + } } sum, err := svc.updateChecksum(txHash) @@ -212,7 +224,7 @@ func (svc *batchService) UpdatePrice(price *big.Int, txHash common.Hash) error { } func (svc *batchService) UpdateBlockNumber(blockNumber uint64) error { - cs := svc.storer.GetChainState() + cs := svc.getChainState() if blockNumber == cs.Block { return nil } @@ -223,17 +235,27 @@ func (svc *batchService) UpdateBlockNumber(blockNumber uint64) error { cs.TotalAmount.Add(cs.TotalAmount, diff.Mul(diff, cs.CurrentPrice)) cs.Block = blockNumber - if err := svc.storer.PutChainState(cs); err != nil { - return fmt.Errorf("put chain state: %w", err) + + if svc.pendingChainState == nil { + if err := svc.storer.PutChainState(cs); err != nil { + return fmt.Errorf("put chain state: %w", err) + } } svc.logger.Debug("block height updated", "new_block", blockNumber) return nil } func (svc *batchService) TransactionStart() error { + svc.pendingChainState = svc.storer.GetChainState() return svc.stateStore.Put(dirtyDBKey, true) } func (svc *batchService) TransactionEnd() error { + if svc.pendingChainState != nil { + if err := svc.storer.PutChainState(svc.pendingChainState); err != nil { + return fmt.Errorf("put chain state: %w", err) + } + svc.pendingChainState = nil + } return svc.stateStore.Delete(dirtyDBKey) } diff --git a/pkg/postage/listener/listener.go b/pkg/postage/listener/listener.go index d9d52b2c5a1..e13314601f2 100644 --- a/pkg/postage/listener/listener.go +++ b/pkg/postage/listener/listener.go @@ -29,6 +29,7 @@ const loggerName = "listener" const ( blockPage = 5000 // how many blocks to sync every time we page + blockPageSnapshot = 50000 // how many blocks to sync every time from snapshot tailSize = 4 // how many blocks to tail from the tip of the chain defaultBatchFactor = uint64(5) // minimal number of blocks to sync at once ) @@ -241,6 +242,15 @@ func (l *listener) Listen(ctx context.Context, from uint64, updater postage.Even l.logger.Debug("batch factor", "value", batchFactor) + // Type assertion to detect if backend is SnapshotLogFilterer + pageSize := uint64(blockPage) + if _, isSnapshot := l.ev.(interface{ GetBatchSnapshot() []byte }); isSnapshot { + pageSize = blockPageSnapshot + l.logger.Debug("using snapshot page size", "page_size", pageSize) + } else { + l.logger.Debug("using standard page size", "page_size", pageSize) + } + synced := make(chan error) closeOnce := new(sync.Once) paged := true @@ -321,9 +331,9 @@ func (l *listener) Listen(ctx context.Context, from uint64, updater postage.Even } // do some paging (sub-optimal) - if to-from >= blockPage { + if to-from >= pageSize { paged = true - to = from + blockPage - 1 + to = from + pageSize - 1 } else { closeOnce.Do(func() { synced <- nil }) } From 1fd77406a427a3577c0563593e10e07eff13f23a Mon Sep 17 00:00:00 2001 From: Calin M Date: Tue, 17 Feb 2026 16:13:03 +0200 Subject: [PATCH 89/96] chore: remove dead use-postage-snapshot flag and unused bootstrapping logic (#5318) --- cmd/bee/cmd/cmd.go | 2 - cmd/bee/cmd/start.go | 1 - packaging/bee.yaml | 2 - packaging/homebrew-amd64/bee.yaml | 2 - packaging/homebrew-arm64/bee.yaml | 2 - packaging/scoop/bee.yaml | 2 - pkg/node/bootstrap.go | 357 ------------------ pkg/node/node.go | 47 +-- pkg/postage/batchservice/batchservice.go | 10 +- pkg/postage/batchservice/batchservice_test.go | 10 +- pkg/postage/interface.go | 16 +- pkg/postage/listener/listener.go | 9 +- pkg/postage/listener/listener_test.go | 135 +------ 13 files changed, 34 insertions(+), 561 deletions(-) delete mode 100644 pkg/node/bootstrap.go diff --git a/cmd/bee/cmd/cmd.go b/cmd/bee/cmd/cmd.go index 9324d58012f..98a6323be31 100644 --- a/cmd/bee/cmd/cmd.go +++ b/cmd/bee/cmd/cmd.go @@ -72,7 +72,6 @@ const ( optionNameStaticNodes = "static-nodes" optionNameAllowPrivateCIDRs = "allow-private-cidrs" optionNameSleepAfter = "sleep-after" - optionNameUsePostageSnapshot = "use-postage-snapshot" optionNameStorageIncentivesEnable = "storage-incentives-enable" optionNameStateStoreCacheCapacity = "statestore-cache-capacity" optionNameTargetNeighborhood = "target-neighborhood" @@ -288,7 +287,6 @@ func (c *command) setAllFlags(cmd *cobra.Command) { cmd.Flags().Bool(optionNamePProfMutex, false, "enable pprof mutex profile") cmd.Flags().StringSlice(optionNameStaticNodes, []string{}, "protect nodes from getting kicked out on bootnode") cmd.Flags().Bool(optionNameAllowPrivateCIDRs, false, "allow to advertise private CIDRs to the public network") - cmd.Flags().Bool(optionNameUsePostageSnapshot, false, "bootstrap node using postage snapshot from the network") cmd.Flags().Bool(optionNameStorageIncentivesEnable, true, "enable storage incentives feature") cmd.Flags().Uint64(optionNameStateStoreCacheCapacity, 100_000, "lru memory caching capacity in number of statestore entries") cmd.Flags().String(optionNameTargetNeighborhood, "", "neighborhood to target in binary format (ex: 111111001) for mining the initial overlay") diff --git a/cmd/bee/cmd/start.go b/cmd/bee/cmd/start.go index b279f893b4f..2268e8549c1 100644 --- a/cmd/bee/cmd/start.go +++ b/cmd/bee/cmd/start.go @@ -331,7 +331,6 @@ func buildBeeNode(ctx context.Context, c *command, cmd *cobra.Command, logger lo TracingEndpoint: tracingEndpoint, TracingServiceName: c.config.GetString(optionNameTracingServiceName), TrxDebugMode: c.config.GetBool(optionNameTransactionDebugMode), - UsePostageSnapshot: c.config.GetBool(optionNameUsePostageSnapshot), WarmupTime: c.config.GetDuration(optionWarmUpTime), WelcomeMessage: c.config.GetString(optionWelcomeMessage), WhitelistedWithdrawalAddress: c.config.GetStringSlice(optionNameWhitelistedWithdrawalAddress), diff --git a/packaging/bee.yaml b/packaging/bee.yaml index 41a6b2a2568..f5354f8e4d4 100644 --- a/packaging/bee.yaml +++ b/packaging/bee.yaml @@ -110,8 +110,6 @@ password-file: "/var/lib/bee/password" # tracing-service-name: bee ## skips the gas estimate step for contract transactions # transaction-debug-mode: false -## bootstrap node using postage snapshot from the network -# use-postage-snapshot: false ## log verbosity level 0=silent, 1=error, 2=warn, 3=info, 4=debug, 5=trace # verbosity: info ## maximum node warmup duration; proceeds when stable or after this time diff --git a/packaging/homebrew-amd64/bee.yaml b/packaging/homebrew-amd64/bee.yaml index 4e93b8e3bab..811ffe62d60 100644 --- a/packaging/homebrew-amd64/bee.yaml +++ b/packaging/homebrew-amd64/bee.yaml @@ -110,8 +110,6 @@ password-file: "/usr/local/var/lib/swarm-bee/password" # tracing-service-name: bee ## skips the gas estimate step for contract transactions # transaction-debug-mode: false -## bootstrap node using postage snapshot from the network -# use-postage-snapshot: false ## log verbosity level 0=silent, 1=error, 2=warn, 3=info, 4=debug, 5=trace # verbosity: info ## maximum node warmup duration; proceeds when stable or after this time diff --git a/packaging/homebrew-arm64/bee.yaml b/packaging/homebrew-arm64/bee.yaml index 291288c3eed..382405cb149 100644 --- a/packaging/homebrew-arm64/bee.yaml +++ b/packaging/homebrew-arm64/bee.yaml @@ -110,8 +110,6 @@ password-file: "/opt/homebrew/var/lib/swarm-bee/password" # tracing-service-name: bee ## skips the gas estimate step for contract transactions # transaction-debug-mode: false -## bootstrap node using postage snapshot from the network -# use-postage-snapshot: false ## log verbosity level 0=silent, 1=error, 2=warn, 3=info, 4=debug, 5=trace # verbosity: info ## maximum node warmup duration; proceeds when stable or after this time diff --git a/packaging/scoop/bee.yaml b/packaging/scoop/bee.yaml index d73c9f4feb1..58b48f70b52 100644 --- a/packaging/scoop/bee.yaml +++ b/packaging/scoop/bee.yaml @@ -110,8 +110,6 @@ password-file: "./password" # tracing-service-name: bee ## skips the gas estimate step for contract transactions # transaction-debug-mode: false -## bootstrap node using postage snapshot from the network -# use-postage-snapshot: false ## log verbosity level 0=silent, 1=error, 2=warn, 3=info, 4=debug, 5=trace # verbosity: info ## maximum node warmup duration; proceeds when stable or after this time diff --git a/pkg/node/bootstrap.go b/pkg/node/bootstrap.go deleted file mode 100644 index eab32166ec9..00000000000 --- a/pkg/node/bootstrap.go +++ /dev/null @@ -1,357 +0,0 @@ -// Copyright 2022 The Swarm Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package node - -import ( - "context" - "crypto/ecdsa" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "io" - "math/big" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethersphere/bee/v2/pkg/accounting" - "github.com/ethersphere/bee/v2/pkg/addressbook" - "github.com/ethersphere/bee/v2/pkg/crypto" - "github.com/ethersphere/bee/v2/pkg/feeds" - "github.com/ethersphere/bee/v2/pkg/feeds/factory" - "github.com/ethersphere/bee/v2/pkg/file" - "github.com/ethersphere/bee/v2/pkg/file/joiner" - "github.com/ethersphere/bee/v2/pkg/file/loadsave" - "github.com/ethersphere/bee/v2/pkg/file/redundancy" - "github.com/ethersphere/bee/v2/pkg/hive" - "github.com/ethersphere/bee/v2/pkg/log" - "github.com/ethersphere/bee/v2/pkg/manifest" - "github.com/ethersphere/bee/v2/pkg/p2p/libp2p" - "github.com/ethersphere/bee/v2/pkg/postage" - "github.com/ethersphere/bee/v2/pkg/pricer" - "github.com/ethersphere/bee/v2/pkg/pricing" - "github.com/ethersphere/bee/v2/pkg/retrieval" - "github.com/ethersphere/bee/v2/pkg/settlement/pseudosettle" - "github.com/ethersphere/bee/v2/pkg/spinlock" - "github.com/ethersphere/bee/v2/pkg/stabilization" - "github.com/ethersphere/bee/v2/pkg/storage" - "github.com/ethersphere/bee/v2/pkg/storer" - "github.com/ethersphere/bee/v2/pkg/swarm" - "github.com/ethersphere/bee/v2/pkg/topology" - "github.com/ethersphere/bee/v2/pkg/topology/kademlia" - "github.com/ethersphere/bee/v2/pkg/topology/lightnode" - "github.com/ethersphere/bee/v2/pkg/tracing" - "github.com/hashicorp/go-multierror" - ma "github.com/multiformats/go-multiaddr" -) - -var ( - // zeroed out while waiting to be replacement for the new snapshot feed address - // must be different to avoid stale reads on the old contract - snapshotFeed = swarm.MustParseHexAddress("0000000000000000000000000000000000000000000000000000000000000000") - errDataMismatch = errors.New("data length mismatch") -) - -const ( - getSnapshotRetries = 3 - retryWait = time.Second * 5 - timeout = time.Minute * 2 -) - -func bootstrapNode( - ctx context.Context, - addr string, - swarmAddress swarm.Address, - nonce []byte, - addressbook addressbook.Interface, - bootnodes []ma.Multiaddr, - lightNodes *lightnode.Container, - stateStore storage.StateStorer, - signer crypto.Signer, - networkID uint64, - logger log.Logger, - libp2pPrivateKey *ecdsa.PrivateKey, - detector *stabilization.Detector, - o *Options, -) (snapshot *postage.ChainSnapshot, retErr error) { - tracer, tracerCloser, err := tracing.NewTracer(&tracing.Options{ - Enabled: o.TracingEnabled, - Endpoint: o.TracingEndpoint, - ServiceName: o.TracingServiceName, - }) - if err != nil { - return nil, fmt.Errorf("tracer: %w", err) - } - - p2pCtx, p2pCancel := context.WithCancel(ctx) - - b := &Bee{ - ctxCancel: p2pCancel, - tracerCloser: tracerCloser, - } - - defer func() { - retErr = multierror.Append(new(multierror.Error), retErr, b.Shutdown()).ErrorOrNil() - }() - - p2ps, err := libp2p.New(p2pCtx, signer, networkID, swarmAddress, addr, addressbook, stateStore, lightNodes, logger, tracer, libp2p.Options{ - PrivateKey: libp2pPrivateKey, - NATAddr: o.NATAddr, - NATWSSAddr: o.NATWSSAddr, - EnableWS: o.EnableWS, - EnableWSS: o.EnableWSS, - WSSAddr: o.WSSAddr, - AutoTLSStorageDir: o.AutoTLSStorageDir, - AutoTLSDomain: o.AutoTLSDomain, - AutoTLSRegistrationEndpoint: o.AutoTLSRegistrationEndpoint, - AutoTLSCAEndpoint: o.AutoTLSCAEndpoint, - WelcomeMessage: o.WelcomeMessage, - FullNode: false, - Nonce: nonce, - }) - if err != nil { - return nil, fmt.Errorf("p2p service: %w", err) - } - b.p2pService = p2ps - b.p2pHalter = p2ps - - hive := hive.New(p2ps, addressbook, networkID, o.BootnodeMode, o.AllowPrivateCIDRs, swarmAddress, logger) - - if err = p2ps.AddProtocol(hive.Protocol()); err != nil { - return nil, fmt.Errorf("hive service: %w", err) - } - b.hiveCloser = hive - - kad, err := kademlia.New(swarmAddress, addressbook, hive, p2ps, detector, logger, - kademlia.Options{Bootnodes: bootnodes, BootnodeMode: o.BootnodeMode, StaticNodes: o.StaticNodes, DataDir: o.DataDir}) - if err != nil { - return nil, fmt.Errorf("unable to create kademlia: %w", err) - } - b.topologyCloser = kad - b.topologyHalter = kad - hive.SetAddPeersHandler(kad.AddPeers) - p2ps.SetPickyNotifier(kad) - - paymentThreshold, _ := new(big.Int).SetString(o.PaymentThreshold, 10) - lightPaymentThreshold := new(big.Int).Div(paymentThreshold, big.NewInt(lightFactor)) - - pricer := pricer.NewFixedPricer(swarmAddress, basePrice) - - pricing := pricing.New(p2ps, logger, paymentThreshold, lightPaymentThreshold, big.NewInt(minPaymentThreshold)) - if err = p2ps.AddProtocol(pricing.Protocol()); err != nil { - return nil, fmt.Errorf("pricing service: %w", err) - } - - acc, err := accounting.NewAccounting( - paymentThreshold, - o.PaymentTolerance, - o.PaymentEarly, - logger, - stateStore, - pricing, - big.NewInt(refreshRate), - lightFactor, - p2ps, - ) - if err != nil { - return nil, fmt.Errorf("accounting: %w", err) - } - b.accountingCloser = acc - - // bootstrapper mode uses the light node refresh rate - enforcedRefreshRate := big.NewInt(lightRefreshRate) - - pseudosettleService := pseudosettle.New(p2ps, logger, stateStore, acc, enforcedRefreshRate, enforcedRefreshRate, p2ps) - if err = p2ps.AddProtocol(pseudosettleService.Protocol()); err != nil { - return nil, fmt.Errorf("pseudosettle service: %w", err) - } - - acc.SetRefreshFunc(pseudosettleService.Pay) - - pricing.SetPaymentThresholdObserver(acc) - - localStore, err := storer.New(ctx, "", &storer.Options{ - CacheCapacity: 1_000_000, - }) - if err != nil { - return nil, fmt.Errorf("local store creation: %w", err) - } - b.localstoreCloser = localStore - - radiusF := func() (uint8, error) { return swarm.MaxBins, nil } - - retrieve := retrieval.New(swarmAddress, radiusF, localStore, p2ps, kad, logger, acc, pricer, tracer, o.RetrievalCaching) - if err = p2ps.AddProtocol(retrieve.Protocol()); err != nil { - return nil, fmt.Errorf("retrieval service: %w", err) - } - b.retrievalCloser = retrieve - - localStore.SetRetrievalService(retrieve) - - if err := kad.Start(p2pCtx); err != nil { - return nil, err - } - - if err := p2ps.Ready(); err != nil { - return nil, err - } - - if err := waitPeers(kad); err != nil { - return nil, errors.New("timed out waiting for kademlia peers") - } - - logger.Info("bootstrap: trying to fetch stamps snapshot") - - var ( - snapshotRootCh swarm.Chunk - reader file.Joiner - l int64 - eventsJSON []byte - ) - - for range getSnapshotRetries { - if err != nil { - time.Sleep(retryWait) - } - - ctx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - - snapshotRootCh, err = getLatestSnapshot(ctx, localStore.Download(true), localStore.Cache(), snapshotFeed) - if err != nil { - logger.Warning("bootstrap: fetching snapshot failed", "error", err) - continue - } - break - } - if err != nil { - return nil, err - } - - for range getSnapshotRetries { - if err != nil { - time.Sleep(retryWait) - } - - ctx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - - reader, l, err = joiner.NewJoiner(ctx, localStore.Download(true), localStore.Cache(), snapshotRootCh.Address(), snapshotRootCh) - if err != nil { - logger.Warning("bootstrap: file joiner failed", "error", err) - continue - } - - eventsJSON, err = io.ReadAll(reader) - if err != nil { - logger.Warning("bootstrap: reading failed", "error", err) - continue - } - - if len(eventsJSON) != int(l) { - err = errDataMismatch - logger.Warning("bootstrap: count mismatch", "error", err) - continue - } - break - } - if err != nil { - return nil, err - } - - events := postage.ChainSnapshot{} - err = json.Unmarshal(eventsJSON, &events) - if err != nil { - return nil, err - } - - return &events, nil -} - -// wait till some peers are connected. returns true if all is ok -func waitPeers(kad *kademlia.Kad) error { - const minPeersCount = 25 - return spinlock.WaitWithInterval(time.Minute, time.Second, func() bool { - count := 0 - _ = kad.EachConnectedPeer(func(_ swarm.Address, _ uint8) (bool, bool, error) { - count++ - return false, false, nil - }, topology.Select{}) - return count >= minPeersCount - }) -} - -func getLatestSnapshot( - ctx context.Context, - st storage.Getter, - putter storage.Putter, - address swarm.Address, -) (swarm.Chunk, error) { - ls := loadsave.NewReadonly(st, putter, redundancy.DefaultLevel) - feedFactory := factory.New(st) - - m, err := manifest.NewDefaultManifestReference( - address, - ls, - ) - if err != nil { - return nil, fmt.Errorf("not a manifest: %w", err) - } - - e, err := m.Lookup(ctx, "/") - if err != nil { - return nil, fmt.Errorf("node lookup: %w", err) - } - - var ( - owner, topic []byte - t = new(feeds.Type) - ) - meta := e.Metadata() - if e := meta["swarm-feed-owner"]; e != "" { - owner, err = hex.DecodeString(e) - if err != nil { - return nil, err - } - } - if e := meta["swarm-feed-topic"]; e != "" { - topic, err = hex.DecodeString(e) - if err != nil { - return nil, err - } - } - if e := meta["swarm-feed-type"]; e != "" { - err := t.FromString(e) - if err != nil { - return nil, err - } - } - if len(owner) == 0 || len(topic) == 0 { - return nil, fmt.Errorf("node lookup: %s", "feed metadata absent") - } - f := feeds.New(topic, common.BytesToAddress(owner)) - - l, err := feedFactory.NewLookup(*t, f) - if err != nil { - return nil, fmt.Errorf("feed lookup failed: %w", err) - } - - u, _, _, err := l.At(ctx, time.Now().Unix(), 0) - if err != nil { - return nil, err - } - - return feeds.GetWrappedChunk(ctx, st, u, false) -} - -func batchStoreExists(s storage.StateStorer) (bool, error) { - hasOne := false - err := s.Iterate("batchstore_", func(key, value []byte) (stop bool, err error) { - hasOne = true - return true, err - }) - - return hasOne, err -} diff --git a/pkg/node/node.go b/pkg/node/node.go index 56cdbc4572f..7564d894f58 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -63,6 +63,7 @@ import ( "github.com/ethersphere/bee/v2/pkg/stabilization" "github.com/ethersphere/bee/v2/pkg/status" "github.com/ethersphere/bee/v2/pkg/steward" + "github.com/ethersphere/bee/v2/pkg/storage" "github.com/ethersphere/bee/v2/pkg/storageincentives" "github.com/ethersphere/bee/v2/pkg/storageincentives/redistribution" "github.com/ethersphere/bee/v2/pkg/storageincentives/staking" @@ -117,7 +118,6 @@ type Bee struct { saludCloser io.Closer storageIncetivesCloser io.Closer pushSyncCloser io.Closer - retrievalCloser io.Closer shutdownInProgress bool shutdownMutex sync.Mutex syncingStopped *syncutil.Signaler @@ -182,7 +182,6 @@ type Options struct { TracingEndpoint string TracingServiceName string TrxDebugMode bool - UsePostageSnapshot bool WarmupTime time.Duration WelcomeMessage string WhitelistedWithdrawalAddress []string @@ -629,34 +628,6 @@ func NewBee( logger.Debug("node warmup check: period complete.", "periodEndTime", t, "eventsInPeriod", periodCount, "rateStdDev", stDev) } - var initBatchState *postage.ChainSnapshot - // Bootstrap node with postage snapshot only if it is running on mainnet, is a fresh - // install or explicitly asked by user to resync - if networkID == mainnetNetworkID && o.UsePostageSnapshot && (!batchStoreExists || o.Resync) { - start := time.Now() - logger.Info("cold postage start detected. fetching postage stamp snapshot from swarm") - initBatchState, err = bootstrapNode( - ctx, - addr, - swarmAddress, - nonce, - addressbook, - bootnodes, - lightNodes, - stateStore, - signer, - networkID, - log.Noop, - libp2pPrivateKey, - detector, - o, - ) - logger.Info("bootstrapper created", "elapsed", time.Since(start)) - if err != nil { - logger.Error(err, "bootstrapper failed to fetch batch state") - } - } - var registry *prometheus.Registry if apiService != nil { @@ -845,7 +816,7 @@ func NewBee( if err != nil { logger.Error(err, "failed to initialize batch service from snapshot, continuing outside snapshot block...") } else { - err = snapshotBatchSvc.Start(ctx, postageSyncStart, initBatchState) + err = snapshotBatchSvc.Start(ctx, postageSyncStart) syncStatus.Store(true) if err != nil { syncErr.Store(err) @@ -873,7 +844,7 @@ func NewBee( } if o.FullNodeMode { - err = batchSvc.Start(ctx, postageSyncStart, initBatchState) + err = batchSvc.Start(ctx, postageSyncStart) syncStatus.Store(true) if err != nil { syncErr.Store(err) @@ -882,7 +853,7 @@ func NewBee( } else { go func() { logger.Info("started postage contract data sync in the background...") - err := batchSvc.Start(ctx, postageSyncStart, initBatchState) + err := batchSvc.Start(ctx, postageSyncStart) syncStatus.Store(true) if err != nil { syncErr.Store(err) @@ -1529,3 +1500,13 @@ func validatePublicAddress(addr string) error { return nil } + +func batchStoreExists(s storage.StateStorer) (bool, error) { + hasOne := false + err := s.Iterate("batchstore_", func(key, value []byte) (stop bool, err error) { + hasOne = true + return true, err + }) + + return hasOne, err +} diff --git a/pkg/postage/batchservice/batchservice.go b/pkg/postage/batchservice/batchservice.go index aac224c1a89..cf3382c552d 100644 --- a/pkg/postage/batchservice/batchservice.go +++ b/pkg/postage/batchservice/batchservice.go @@ -261,14 +261,14 @@ func (svc *batchService) TransactionEnd() error { var ErrInterruped = errors.New("postage sync interrupted") -func (svc *batchService) Start(ctx context.Context, startBlock uint64, initState *postage.ChainSnapshot) (err error) { +func (svc *batchService) Start(ctx context.Context, startBlock uint64) (err error) { dirty := false err = svc.stateStore.Get(dirtyDBKey, &dirty) if err != nil && !errors.Is(err, storage.ErrNotFound) { return err } - if dirty || svc.resync || initState != nil { + if dirty || svc.resync { if dirty { svc.logger.Warning("batch service: dirty shutdown detected, resetting batch store") @@ -290,11 +290,7 @@ func (svc *batchService) Start(ctx context.Context, startBlock uint64, initState startBlock = cs.Block } - if initState != nil && initState.LastBlockNumber > startBlock { - startBlock = initState.LastBlockNumber - } - - syncedChan := svc.listener.Listen(ctx, startBlock+1, svc, initState) + syncedChan := svc.listener.Listen(ctx, startBlock+1, svc) return <-syncedChan } diff --git a/pkg/postage/batchservice/batchservice_test.go b/pkg/postage/batchservice/batchservice_test.go index 9500acaba28..2a9ef622c10 100644 --- a/pkg/postage/batchservice/batchservice_test.go +++ b/pkg/postage/batchservice/batchservice_test.go @@ -32,7 +32,7 @@ var ( type mockListener struct { } -func (*mockListener) Listen(ctx context.Context, from uint64, updater postage.EventUpdater, _ *postage.ChainSnapshot) <-chan error { +func (*mockListener) Listen(ctx context.Context, from uint64, updater postage.EventUpdater) <-chan error { c := make(chan error, 1) c <- nil return c @@ -514,7 +514,7 @@ func TestTransactionOk(t *testing.T) { t.Parallel() svc, store, s := newTestStoreAndService(t) - if err := svc.Start(context.Background(), 10, nil); err != nil { + if err := svc.Start(context.Background(), 10); err != nil { t.Fatal(err) } @@ -530,7 +530,7 @@ func TestTransactionOk(t *testing.T) { if err != nil { t.Fatal(err) } - if err := svc2.Start(context.Background(), 10, nil); err != nil { + if err := svc2.Start(context.Background(), 10); err != nil { t.Fatal(err) } @@ -543,7 +543,7 @@ func TestTransactionError(t *testing.T) { t.Parallel() svc, store, s := newTestStoreAndService(t) - if err := svc.Start(context.Background(), 10, nil); err != nil { + if err := svc.Start(context.Background(), 10); err != nil { t.Fatal(err) } @@ -555,7 +555,7 @@ func TestTransactionError(t *testing.T) { if err != nil { t.Fatal(err) } - if err := svc2.Start(context.Background(), 10, nil); err != nil { + if err := svc2.Start(context.Background(), 10); err != nil { t.Fatal(err) } diff --git a/pkg/postage/interface.go b/pkg/postage/interface.go index 2288fb9fcbf..0f8c9f976d6 100644 --- a/pkg/postage/interface.go +++ b/pkg/postage/interface.go @@ -10,7 +10,6 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" ) // EventUpdater interface definitions reflect the updates triggered by events @@ -21,23 +20,12 @@ type EventUpdater interface { UpdateDepth(id []byte, depth uint8, normalisedBalance *big.Int, txHash common.Hash) error UpdatePrice(price *big.Int, txHash common.Hash) error UpdateBlockNumber(blockNumber uint64) error - Start(ctx context.Context, startBlock uint64, initState *ChainSnapshot) error + Start(ctx context.Context, startBlock uint64) error TransactionStart() error TransactionEnd() error } -// ChainSnapshot represents the snapshot of all the postage events between the -// FirstBlockNumber and LastBlockNumber. The timestamp stores the time at which the -// snapshot was generated. This snapshot can be used to sync the postage package -// to prevent large no. of chain backend calls. -type ChainSnapshot struct { - Events []types.Log `json:"events"` - LastBlockNumber uint64 `json:"lastBlockNumber"` - FirstBlockNumber uint64 `json:"firstBlockNumber"` - Timestamp int64 `json:"timestamp"` -} - // Storer represents the persistence layer for batches // on the current (highest available) block. type Storer interface { @@ -90,7 +78,7 @@ type ChainStateGetter interface { // Listener provides a blockchain event iterator. type Listener interface { io.Closer - Listen(ctx context.Context, from uint64, updater EventUpdater, initState *ChainSnapshot) <-chan error + Listen(ctx context.Context, from uint64, updater EventUpdater) <-chan error } type BatchEventListener interface { diff --git a/pkg/postage/listener/listener.go b/pkg/postage/listener/listener.go index e13314601f2..53feb3a5299 100644 --- a/pkg/postage/listener/listener.go +++ b/pkg/postage/listener/listener.go @@ -187,7 +187,7 @@ func (l *listener) processEvent(e types.Log, updater postage.EventUpdater) error } } -func (l *listener) Listen(ctx context.Context, from uint64, updater postage.EventUpdater, initState *postage.ChainSnapshot) <-chan error { +func (l *listener) Listen(ctx context.Context, from uint64, updater postage.EventUpdater) <-chan error { ctx, cancel := context.WithCancel(ctx) go func() { <-l.quit @@ -227,13 +227,6 @@ func (l *listener) Listen(ctx context.Context, from uint64, updater postage.Even return nil } - if initState != nil { - err := processEvents(initState.Events, initState.LastBlockNumber+1) - if err != nil { - l.logger.Error(err, "failed bootstrapping from initial state") - } - } - batchFactor, err := strconv.ParseUint(batchFactorOverridePublic, 10, 64) if err != nil { l.logger.Warning("batch factor conversation failed", "batch_factor", batchFactor, "error", err) diff --git a/pkg/postage/listener/listener_test.go b/pkg/postage/listener/listener_test.go index bd15c2659ab..bb34086bb6a 100644 --- a/pkg/postage/listener/listener_test.go +++ b/pkg/postage/listener/listener_test.go @@ -18,7 +18,6 @@ import ( "github.com/ethereum/go-ethereum/core/types" chaincfg "github.com/ethersphere/bee/v2/pkg/config" "github.com/ethersphere/bee/v2/pkg/log" - "github.com/ethersphere/bee/v2/pkg/postage" "github.com/ethersphere/bee/v2/pkg/postage/listener" "github.com/ethersphere/bee/v2/pkg/util/abiutil" "github.com/ethersphere/bee/v2/pkg/util/syncutil" @@ -81,7 +80,7 @@ func TestListener(t *testing.T) { backoffTime, ) testutil.CleanupCloser(t, l) - <-l.Listen(context.Background(), 0, ev, nil) + <-l.Listen(context.Background(), 0, ev) select { case e := <-ev.eventC: @@ -122,7 +121,7 @@ func TestListener(t *testing.T) { backoffTime, ) testutil.CleanupCloser(t, l) - <-l.Listen(context.Background(), 0, ev, nil) + <-l.Listen(context.Background(), 0, ev) select { case e := <-ev.eventC: @@ -164,7 +163,7 @@ func TestListener(t *testing.T) { ) testutil.CleanupCloser(t, l) - <-l.Listen(context.Background(), 0, ev, nil) + <-l.Listen(context.Background(), 0, ev) select { case e := <-ev.eventC: @@ -203,7 +202,7 @@ func TestListener(t *testing.T) { backoffTime, ) testutil.CleanupCloser(t, l) - <-l.Listen(context.Background(), 0, ev, nil) + <-l.Listen(context.Background(), 0, ev) select { case e := <-ev.eventC: @@ -266,7 +265,7 @@ func TestListener(t *testing.T) { backoffTime, ) testutil.CleanupCloser(t, l) - <-l.Listen(context.Background(), 0, ev, nil) + <-l.Listen(context.Background(), 0, ev) select { case e := <-ev.eventC: @@ -346,7 +345,7 @@ func TestListener(t *testing.T) { 0, ) testutil.CleanupCloser(t, l) - <-l.Listen(context.Background(), 0, ev, nil) + <-l.Listen(context.Background(), 0, ev) select { case e := <-ev.eventC: @@ -374,7 +373,7 @@ func TestListener(t *testing.T) { 0, ) testutil.CleanupCloser(t, l) - <-l.Listen(context.Background(), 0, ev, nil) + <-l.Listen(context.Background(), 0, ev) select { case <-c.C: @@ -401,7 +400,7 @@ func TestListener(t *testing.T) { backoffTime, ) testutil.CleanupCloser(t, l) - <-l.Listen(context.Background(), 0, ev, nil) + <-l.Listen(context.Background(), 0, ev) select { case e := <-ev.eventC: @@ -418,122 +417,6 @@ func TestListener(t *testing.T) { }) } -func TestListenerBatchState(t *testing.T) { - t.Parallel() - - ev := newEventUpdaterMock() - mf := newMockFilterer() - - create := createArgs{ - id: hash[:], - owner: addr[:], - amount: big.NewInt(42), - normalisedAmount: big.NewInt(43), - depth: 100, - } - - topup := topupArgs{ - id: hash[:], - amount: big.NewInt(0), - normalisedBalance: big.NewInt(1), - } - - depthIncrease := depthArgs{ - id: hash[:], - depth: 200, - normalisedBalance: big.NewInt(2), - } - - priceUpdate := priceArgs{ - price: big.NewInt(500), - } - - snapshot := &postage.ChainSnapshot{ - Events: []types.Log{ - create.toLog(496), - topup.toLog(497), - depthIncrease.toLog(498), - priceUpdate.toLog(499), - }, - FirstBlockNumber: 496, - LastBlockNumber: 499, - Timestamp: time.Now().Unix(), - } - - stop := make(chan struct{}) - done := make(chan struct{}) - errs := make(chan error) - noOfEvents := 0 - - go func() { - for { - select { - case <-stop: - return - case e := <-ev.eventC: - noOfEvents++ - switch ev := e.(type) { - case blockNumberCall: - if ev.blockNumber < 497 && ev.blockNumber > 500 { - errs <- fmt.Errorf("invalid blocknumber call %d", ev.blockNumber) - return - } - if ev.blockNumber == 500 { - close(done) - return - } - case createArgs: - if err := ev.compare(create); err != nil { - errs <- err - return - } - case topupArgs: - if err := ev.compare(topup); err != nil { - errs <- err - return - } - case depthArgs: - if err := ev.compare(depthIncrease); err != nil { - errs <- err - return - } - case priceArgs: - if err := ev.compare(priceUpdate); err != nil { - errs <- err - return - } - } - } - } - }() - - l := listener.New( - nil, - log.Noop, - mf, - postageStampContractAddress, - postageStampContractABI, - 1, - stallingTimeout, - backoffTime, - ) - testutil.CleanupCloser(t, l) - l.Listen(context.Background(), snapshot.LastBlockNumber+1, ev, snapshot) - - defer close(stop) - - select { - case <-time.After(5 * time.Second): - t.Fatal("timedout waiting for events to be processed", noOfEvents) - case err := <-errs: - t.Fatal(err) - case <-done: - if noOfEvents != 9 { - t.Fatal("invalid count of events on completion", noOfEvents) - } - } -} - func newEventUpdaterMock() *updater { return &updater{ eventC: make(chan any, 1), @@ -595,7 +478,7 @@ func (u *updater) UpdateBlockNumber(blockNumber uint64) error { return u.blockNumberUpdateError } -func (u *updater) Start(ctx context.Context, bno uint64, cs *postage.ChainSnapshot) error { +func (u *updater) Start(ctx context.Context, bno uint64) error { return nil } From bb053081f1200674690af84df8751345128dfd2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ljubi=C5=A1a=20Ga=C4=8Devi=C4=87?= <35105035+gacevicljubisa@users.noreply.github.com> Date: Thu, 19 Feb 2026 12:22:17 +0100 Subject: [PATCH 90/96] fix(p2p): add exponential backoff for reacher (#5371) --- pkg/p2p/libp2p/internal/reacher/metrics.go | 56 ++++ pkg/p2p/libp2p/internal/reacher/reacher.go | 95 +++++- .../libp2p/internal/reacher/reacher_test.go | 278 +++++++++++++++++- pkg/p2p/libp2p/metrics.go | 6 +- 4 files changed, 420 insertions(+), 15 deletions(-) create mode 100644 pkg/p2p/libp2p/internal/reacher/metrics.go diff --git a/pkg/p2p/libp2p/internal/reacher/metrics.go b/pkg/p2p/libp2p/internal/reacher/metrics.go new file mode 100644 index 00000000000..a288f2fa8d5 --- /dev/null +++ b/pkg/p2p/libp2p/internal/reacher/metrics.go @@ -0,0 +1,56 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package reacher + +import ( + m "github.com/ethersphere/bee/v2/pkg/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +// metrics groups reacher related prometheus counters. +type metrics struct { + Peers prometheus.Gauge + PingAttemptCount prometheus.Counter + PingErrorCount prometheus.Counter + PingDuration prometheus.Histogram +} + +// newMetrics is a convenient constructor for creating new metrics. +func newMetrics() metrics { + const subsystem = "reacher" + + return metrics{ + Peers: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: m.Namespace, + Subsystem: subsystem, + Name: "peers", + Help: "Number of peers currently in the reacher queue.", + }), + PingAttemptCount: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: m.Namespace, + Subsystem: subsystem, + Name: "ping_attempt_count", + Help: "Number of ping attempts.", + }), + PingErrorCount: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: m.Namespace, + Subsystem: subsystem, + Name: "ping_error_count", + Help: "Number of failed ping attempts.", + }), + PingDuration: prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: m.Namespace, + Subsystem: subsystem, + Name: "ping_duration_seconds", + Help: "Ping latency distribution in seconds.", + Buckets: []float64{.1, .25, .5, 1, 2, 5, 10, 15}, + }), + } +} + +// Metrics returns set of prometheus collectors. +func (r *reacher) Metrics() []prometheus.Collector { + return m.PrometheusCollectorsFromFields(r.metrics) +} diff --git a/pkg/p2p/libp2p/internal/reacher/reacher.go b/pkg/p2p/libp2p/internal/reacher/reacher.go index 5bdc36c47f2..966b5978411 100644 --- a/pkg/p2p/libp2p/internal/reacher/reacher.go +++ b/pkg/p2p/libp2p/internal/reacher/reacher.go @@ -9,6 +9,7 @@ package reacher import ( "container/heap" "context" + "math/rand/v2" "sync" "time" @@ -19,16 +20,22 @@ import ( ) const ( - pingTimeout = time.Second * 15 - workers = 16 - retryAfterDuration = time.Minute * 5 + pingTimeout = time.Second * 15 + workers = 8 + retryAfterDuration = time.Minute * 5 + maxFailBackoffExponent = 4 // caps failure backoff at retryAfterDuration * 2^4 = 80 min + maxSuccessBackoffExponent = 2 // caps success backoff at retryAfterDuration * 2^2 = 20 min + jitterFactor = 0.2 // ±20% randomization on retry intervals ) type peer struct { - overlay swarm.Address - addr ma.Multiaddr - retryAfter time.Time - index int // index in the heap + overlay swarm.Address + addr ma.Multiaddr + retryAfter time.Time + failCount int // consecutive ping failures for exponential backoff + successCount int // consecutive ping successes for exponential backoff + generation int // incremented on reconnect; guards against stale notifyResult + index int // index in the heap } type reacher struct { @@ -44,6 +51,7 @@ type reacher struct { wg sync.WaitGroup + metrics metrics options *Options logger log.Logger } @@ -52,6 +60,7 @@ type Options struct { PingTimeout time.Duration Workers int RetryAfterDuration time.Duration + JitterFactor float64 // ±N% randomization on retry intervals; 0 disables jitter } func New(streamer p2p.Pinger, notifier p2p.ReachableNotifier, o *Options, log log.Logger) *reacher { @@ -62,6 +71,7 @@ func New(streamer p2p.Pinger, notifier p2p.ReachableNotifier, o *Options, log lo peerHeap: make(peerHeap, 0), peerIndex: make(map[string]*peer), notifier: notifier, + metrics: newMetrics(), logger: log.WithName("reacher").Register(), } @@ -70,6 +80,7 @@ func New(streamer p2p.Pinger, notifier p2p.ReachableNotifier, o *Options, log lo PingTimeout: pingTimeout, Workers: workers, RetryAfterDuration: retryAfterDuration, + JitterFactor: jitterFactor, } } r.options = o @@ -95,7 +106,6 @@ func (r *reacher) manage() { } for { - p, ok, tryAfter := r.tryAcquirePeer() // if no peer is returned, @@ -136,15 +146,22 @@ func (r *reacher) ping(c chan peer, ctx context.Context) { defer r.wg.Done() for p := range c { func() { + r.metrics.PingAttemptCount.Inc() ctxt, cancel := context.WithTimeout(ctx, r.options.PingTimeout) defer cancel() + start := time.Now() rtt, err := r.pinger.Ping(ctxt, p.addr) if err != nil { + r.metrics.PingDuration.Observe(time.Since(start).Seconds()) + r.metrics.PingErrorCount.Inc() r.logger.Debug("ping failed", "peer", p.overlay.String(), "addr", p.addr.String(), "error", err) r.notifier.Reachable(p.overlay, p2p.ReachabilityStatusPrivate) + r.notifyResult(p.overlay, false, p.generation) } else { + r.metrics.PingDuration.Observe(rtt.Seconds()) r.logger.Debug("ping succeeded", "peer", p.overlay.String(), "addr", p.addr.String(), "rtt", rtt) r.notifier.Reachable(p.overlay, p2p.ReachabilityStatusPublic) + r.notifyResult(p.overlay, true, p.generation) } }() } @@ -168,8 +185,10 @@ func (r *reacher) tryAcquirePeer() (peer, bool, time.Duration) { return peer{}, false, time.Until(p.retryAfter) } - // Update retryAfter and fix heap position - p.retryAfter = time.Now().Add(r.options.RetryAfterDuration) + // Set a temporary far-future retryAfter to prevent the manage loop from + // re-dispatching this peer while the ping is in flight. The actual + // retryAfter will be set by notifyResult after the ping completes. + p.retryAfter = now.Add(time.Hour) heap.Fix(&r.peerHeap, p.index) // Return a copy so callers can read fields without holding the lock. @@ -190,13 +209,57 @@ func (r *reacher) Connected(overlay swarm.Address, addr ma.Multiaddr) { if existing, ok := r.peerIndex[key]; ok { existing.addr = addr // Update address for reconnecting peer existing.retryAfter = time.Time{} // Reset to trigger immediate re-ping + existing.failCount = 0 // Fresh start on reconnect + existing.successCount = 0 // Fresh start on reconnect + existing.generation++ // invalidate any in-flight notifyResult heap.Fix(&r.peerHeap, existing.index) } else { p := &peer{overlay: overlay, addr: addr} r.peerIndex[key] = p heap.Push(&r.peerHeap, p) + r.metrics.Peers.Inc() + } + + select { + case r.newPeer <- struct{}{}: + default: + } +} + +// notifyResult updates the peer's retry schedule based on the ping outcome. +// Both success and failure use exponential backoff with different caps: +// - Success: 5m → 10m → 20m (capped at 2^2), resets failCount +// - Failure: 5m → 10m → 20m → 40m → 80m (capped at 2^4), resets successCount +// +// The gen parameter is the generation captured when the ping was dispatched. +// If the peer was reconnected (generation incremented) while the ping was +// in flight, the stale result is discarded. +func (r *reacher) notifyResult(overlay swarm.Address, success bool, gen int) { + r.mu.Lock() + defer r.mu.Unlock() + + p, ok := r.peerIndex[overlay.ByteString()] + if !ok { + return // peer was disconnected while ping was in flight + } + if p.generation != gen { + return // peer was reconnected; discard stale result + } + + if success { + p.failCount = 0 + p.successCount++ + backoff := min(p.successCount, maxSuccessBackoffExponent) + p.retryAfter = time.Now().Add(r.jitter(r.options.RetryAfterDuration * time.Duration(1< Date: Thu, 19 Feb 2026 12:22:45 +0100 Subject: [PATCH 91/96] chore: bump version of OpenAPI to 7.4.1 (#5375) --- openapi/Swarm.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openapi/Swarm.yaml b/openapi/Swarm.yaml index 4d8be527b50..4168e63f80e 100644 --- a/openapi/Swarm.yaml +++ b/openapi/Swarm.yaml @@ -1,7 +1,7 @@ openapi: 3.0.3 info: - version: 7.4.0 + version: 7.4.1 title: Bee API description: "API endpoints for interacting with the Swarm network, supporting file operations, messaging, and node management" From 858f52c76b85578e3334e0b172c7aebdc420596c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ljubi=C5=A1a=20Ga=C4=8Devi=C4=87?= <35105035+gacevicljubisa@users.noreply.github.com> Date: Thu, 19 Feb 2026 15:00:25 +0100 Subject: [PATCH 92/96] fix(p2p): skip addressbook and reacher for unreachable addresses (#5370) --- pkg/p2p/libp2p/connections_test.go | 122 +++++++++++++++++++++++++++++ pkg/p2p/libp2p/export_test.go | 8 ++ pkg/p2p/libp2p/libp2p.go | 54 +++++++++---- 3 files changed, 170 insertions(+), 14 deletions(-) diff --git a/pkg/p2p/libp2p/connections_test.go b/pkg/p2p/libp2p/connections_test.go index 27ba4b7549d..46ecca268e8 100644 --- a/pkg/p2p/libp2p/connections_test.go +++ b/pkg/p2p/libp2p/connections_test.go @@ -13,6 +13,7 @@ import ( "reflect" "strings" "sync" + "sync/atomic" "testing" "time" @@ -27,6 +28,7 @@ import ( "github.com/ethersphere/bee/v2/pkg/statestore/mock" "github.com/ethersphere/bee/v2/pkg/swarm" "github.com/ethersphere/bee/v2/pkg/topology/lightnode" + libp2pcrypto "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/p2p/host/eventbus" libp2pm "github.com/libp2p/go-libp2p" @@ -34,6 +36,7 @@ import ( "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/network" libp2ppeer "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/peerstore" bhost "github.com/libp2p/go-libp2p/p2p/host/basic" swarmt "github.com/libp2p/go-libp2p/p2p/net/swarm/testing" ma "github.com/multiformats/go-multiaddr" @@ -1485,3 +1488,122 @@ var ( noopReachability = func(p2p.ReachabilityStatus) {} noopReachable = func(swarm.Address, p2p.ReachabilityStatus) {} ) + +func TestPeerMultiaddrsNoFallback(t *testing.T) { + t.Parallel() + + s1, _ := newService(t, 1, libp2pServiceOpts{}) + + privKey, _, err := libp2pcrypto.GenerateEd25519Key(rand.New(rand.NewSource(time.Now().UnixNano()))) + if err != nil { + t.Fatal(err) + } + unknownPeerID, err := libp2ppeer.IDFromPrivateKey(privKey) + if err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + addrs, err := s1.PeerMultiaddrs(ctx, unknownPeerID) + if err != nil { + t.Fatal(err) + } + if len(addrs) != 0 { + t.Fatalf("expected no addresses for unknown peer, got %v", addrs) + } +} + +type emptyAddrsPeerstore struct { + peerstore.Peerstore + targetPeerID libp2ppeer.ID + mu sync.RWMutex +} + +func (p *emptyAddrsPeerstore) setTarget(id libp2ppeer.ID) { + p.mu.Lock() + defer p.mu.Unlock() + p.targetPeerID = id +} + +func (p *emptyAddrsPeerstore) isTarget(id libp2ppeer.ID) bool { + p.mu.RLock() + defer p.mu.RUnlock() + return p.targetPeerID == id +} + +func (p *emptyAddrsPeerstore) Addrs(id libp2ppeer.ID) []ma.Multiaddr { + if p.isTarget(id) { + return nil + } + return p.Peerstore.Addrs(id) +} + +func (p *emptyAddrsPeerstore) AddrStream(ctx context.Context, id libp2ppeer.ID) <-chan ma.Multiaddr { + if p.isTarget(id) { + ch := make(chan ma.Multiaddr) + go func() { + <-ctx.Done() + close(ch) + }() + return ch + } + return p.Peerstore.AddrStream(ctx, id) +} + +type emptyAddrsHost struct { + host.Host + ps *emptyAddrsPeerstore +} + +func (h *emptyAddrsHost) Peerstore() peerstore.Peerstore { + return h.ps +} + +func TestConnectEmptyPeerstoreSkipsAddressbookAndReacher(t *testing.T) { + t.Parallel() + + ab1 := addressbook.New(mock.NewStateStore()) + + s1, _ := newService(t, 1, libp2pServiceOpts{ + Addressbook: ab1, + libp2pOpts: libp2p.Options{ + FullNode: true, + }, + }) + + psWrapper := &emptyAddrsPeerstore{Peerstore: s1.Host().Peerstore()} + + var reachableCalled atomic.Bool + notifier1 := mockNotifier(noopCf, noopDf, true) + notifier1.(*notifiee).reachable = func(_ swarm.Address, _ p2p.ReachabilityStatus) { + reachableCalled.Store(true) + } + s1.SetPickyNotifier(notifier1) + + s2, overlay2 := newService(t, 1, libp2pServiceOpts{ + libp2pOpts: libp2p.Options{ + FullNode: true, + }, + }) + + psWrapper.setTarget(s2.Host().ID()) + s1.SetHost(&emptyAddrsHost{Host: s1.Host(), ps: psWrapper}) + + _, err := s2.Connect(context.Background(), serviceUnderlayAddress(t, s1)) + if err != nil { + t.Fatal(err) + } + + expectPeersEventually(t, s1, overlay2) + + _, err = ab1.Get(overlay2) + if err == nil { + t.Fatal("expected addressbook to have no entry for NAT peer, but found one") + } + + if reachableCalled.Load() { + t.Fatal("expected reacher not to be notified for NAT peer") + } +} diff --git a/pkg/p2p/libp2p/export_test.go b/pkg/p2p/libp2p/export_test.go index 7d7db8d5ee4..3f56f7cc16e 100644 --- a/pkg/p2p/libp2p/export_test.go +++ b/pkg/p2p/libp2p/export_test.go @@ -32,6 +32,10 @@ func (s *Service) Host() host.Host { return s.host } +func (s *Service) SetHost(h host.Host) { + s.host = h +} + type StaticAddressResolver = staticAddressResolver var ( @@ -63,6 +67,10 @@ func (s *Service) FilterSupportedAddresses(addrs []ma.Multiaddr) []ma.Multiaddr return s.filterSupportedAddresses(addrs) } +func (s *Service) PeerMultiaddrs(ctx context.Context, peerID libp2ppeer.ID) ([]ma.Multiaddr, error) { + return s.peerMultiaddrs(ctx, peerID) +} + func (s *Service) SetTransportFlags(hasTCP, hasWS, hasWSS bool) { s.enabledTransports = map[bzz.TransportType]bool{ bzz.TransportTCP: hasTCP, diff --git a/pkg/p2p/libp2p/libp2p.go b/pkg/p2p/libp2p/libp2p.go index aae05dc45e0..9b696cd26e2 100644 --- a/pkg/p2p/libp2p/libp2p.go +++ b/pkg/p2p/libp2p/libp2p.go @@ -584,7 +584,7 @@ func (s *Service) handleIncoming(stream network.Stream) { peerID := stream.Conn().RemotePeer() handshakeStream := newStream(stream, s.metrics) - peerMultiaddrs, err := s.peerMultiaddrs(s.ctx, stream.Conn().RemoteMultiaddr(), peerID) + peerAddrs, err := s.peerMultiaddrs(s.ctx, peerID) if err != nil { s.logger.Debug("stream handler: handshake: build remote multiaddrs", "peer_id", peerID, "error", err) s.logger.Error(nil, "stream handler: handshake: build remote multiaddrs", "peer_id", peerID) @@ -593,12 +593,28 @@ func (s *Service) handleIncoming(stream network.Stream) { return } + // For the handshake we always need an observed address (ObservedUnderlay). + // If the peerstore had no addresses, fall back to RemoteMultiaddr for the + // handshake only. This typically means the peer is behind NAT and its + // address is not reachable from the outside. + observedAddrs := peerAddrs + if len(observedAddrs) == 0 { + observedAddrs, err = buildFullMAs([]ma.Multiaddr{stream.Conn().RemoteMultiaddr()}, peerID) + if err != nil { + s.logger.Debug("stream handler: handshake: build remote multiaddrs fallback", "peer_id", peerID, "error", err) + s.logger.Error(nil, "stream handler: handshake: build remote multiaddrs fallback", "peer_id", peerID) + _ = handshakeStream.Reset() + _ = s.host.Network().ClosePeer(peerID) + return + } + } + bee260Compat := s.bee260BackwardCompatibility(peerID) i, err := s.handshakeService.Handle( s.ctx, handshakeStream, - peerMultiaddrs, + observedAddrs, handshake.WithBee260Compatibility(bee260Compat), ) if err != nil { @@ -644,7 +660,8 @@ func (s *Service) handleIncoming(stream network.Stream) { return } - if i.FullNode { + // Only persist in addressbook when we have real peerstore addresses. + if i.FullNode && len(peerAddrs) > 0 { err = s.addressbook.Put(i.BzzAddress.Overlay, *i.BzzAddress) if err != nil { s.logger.Debug("stream handler: addressbook put error", "peer_id", peerID, "error", err) @@ -727,7 +744,9 @@ func (s *Service) handleIncoming(stream network.Stream) { return } - s.notifyReacherConnected(overlay, peerMultiaddrs) + if len(peerAddrs) > 0 { + s.notifyReacherConnected(overlay, peerAddrs) + } peerUserAgent := appendSpace(s.peerUserAgent(s.ctx, peerID)) s.networkStatus.Store(int32(p2p.NetworkStatusAvailable)) @@ -1029,19 +1048,29 @@ func (s *Service) Connect(ctx context.Context, addrs []ma.Multiaddr) (address *b handshakeStream := newStream(stream, s.metrics) - peerMultiaddrs, err := s.peerMultiaddrs(ctx, stream.Conn().RemoteMultiaddr(), peerID) + peerAddrs, err := s.peerMultiaddrs(ctx, peerID) if err != nil { _ = handshakeStream.Reset() _ = s.host.Network().ClosePeer(peerID) return nil, fmt.Errorf("build peer multiaddrs: %w", err) } + observedAddrs := peerAddrs + if len(observedAddrs) == 0 { + observedAddrs, err = buildFullMAs([]ma.Multiaddr{stream.Conn().RemoteMultiaddr()}, peerID) + if err != nil { + _ = handshakeStream.Reset() + _ = s.host.Network().ClosePeer(peerID) + return nil, fmt.Errorf("build peer multiaddrs fallback: %w", err) + } + } + bee260Compat := s.bee260BackwardCompatibility(peerID) i, err := s.handshakeService.Handshake( s.ctx, handshakeStream, - peerMultiaddrs, + observedAddrs, handshake.WithBee260Compatibility(bee260Compat), ) if err != nil { @@ -1120,7 +1149,9 @@ func (s *Service) Connect(ctx context.Context, addrs []ma.Multiaddr) (address *b s.metrics.CreatedConnectionCount.Inc() - s.notifyReacherConnected(overlay, peerMultiaddrs) + if len(peerAddrs) > 0 { + s.notifyReacherConnected(overlay, peerAddrs) + } peerUA := appendSpace(s.peerUserAgent(ctx, peerID)) loggerV1.Debug("successfully connected to peer (outbound)", "addresses", i.BzzAddress.ShortString(), "light", i.LightString(), "user_agent", peerUA) @@ -1429,17 +1460,12 @@ func (s *Service) determineCurrentNetworkStatus(err error) error { return err } -// peerMultiaddrs builds full multiaddresses for a peer given information from -// the libp2p host peerstore. If the peerstore doesn't have addresses yet, -// it falls back to using the remote address from the active connection. -func (s *Service) peerMultiaddrs(ctx context.Context, remoteAddr ma.Multiaddr, peerID libp2ppeer.ID) ([]ma.Multiaddr, error) { +// peerMultiaddrs builds full multiaddresses for a peer using the peerstore. +func (s *Service) peerMultiaddrs(ctx context.Context, peerID libp2ppeer.ID) ([]ma.Multiaddr, error) { waitPeersCtx, cancel := context.WithTimeout(ctx, peerstoreWaitAddrsTimeout) defer cancel() mas := waitPeerAddrs(waitPeersCtx, s.host.Peerstore(), peerID) - if len(mas) == 0 && remoteAddr != nil { - mas = []ma.Multiaddr{remoteAddr} - } return buildFullMAs(mas, peerID) } From f27d3f9ea3d15cf1545ddb0181631e35cd9ed49b Mon Sep 17 00:00:00 2001 From: acud <12988138+acud@users.noreply.github.com> Date: Tue, 24 Feb 2026 11:27:44 -0600 Subject: [PATCH 93/96] test: reserve sample deterministic test + benchmarks (#5374) --- pkg/accesscontrol/access_test.go | 1 - pkg/storer/sample_test.go | 157 ++++++++++++++++++++++++++++++- 2 files changed, 155 insertions(+), 3 deletions(-) diff --git a/pkg/accesscontrol/access_test.go b/pkg/accesscontrol/access_test.go index 0b2eeaa1e56..082a995ad03 100644 --- a/pkg/accesscontrol/access_test.go +++ b/pkg/accesscontrol/access_test.go @@ -71,7 +71,6 @@ func getPrivKey(keyNumber int) *ecdsa.PrivateKey { } func TestDecryptRef_Publisher(t *testing.T) { - t.Parallel() ctx := context.Background() id1 := getPrivKey(1) s := kvsmock.New() diff --git a/pkg/storer/sample_test.go b/pkg/storer/sample_test.go index 49745c4b24f..a73ebbcf96e 100644 --- a/pkg/storer/sample_test.go +++ b/pkg/storer/sample_test.go @@ -6,10 +6,12 @@ package storer_test import ( "context" + "fmt" "math/rand" "testing" "time" + "github.com/ethersphere/bee/v2/pkg/cac" "github.com/ethersphere/bee/v2/pkg/postage" postagetesting "github.com/ethersphere/bee/v2/pkg/postage/testing" @@ -112,7 +114,6 @@ func TestReserveSampler(t *testing.T) { assertSampleNoErrors(t, sample) }) - } t.Run("disk", func(t *testing.T) { @@ -233,7 +234,6 @@ func TestReserveSamplerSisterNeighborhood(t *testing.T) { t.Fatalf("sample should not have ignored chunks") } }) - } t.Run("disk", func(t *testing.T) { @@ -309,6 +309,63 @@ func assertValidSample(t *testing.T, sample storer.Sample, minRadius uint8, anch } } +// TestSampleVectorCAC is a deterministic test vector that verifies the chunk +// address and transformed address produced by MakeSampleUsingChunks for a +// single hardcoded CAC chunk and anchor. It guards against regressions in the +// BMT hashing or sampling pipeline. +func TestSampleVectorCAC(t *testing.T) { + t.Parallel() + + // Chunk content: 4096 bytes with repeating pattern i%256. + chunkContent := make([]byte, swarm.ChunkSize) + for i := range chunkContent { + chunkContent[i] = byte(i % 256) + } + + ch, err := cac.New(chunkContent) + if err != nil { + t.Fatal(err) + } + + // Attach a hardcoded (but otherwise irrelevant) stamp so that + // MakeSampleUsingChunks can read ch.Stamp() without panicking. + batchID := make([]byte, 32) + for i := range batchID { + batchID[i] = byte(i + 1) + } + sig := make([]byte, 65) + for i := range sig { + sig[i] = byte(i + 1) + } + ch = ch.WithStamp(postage.NewStamp(batchID, make([]byte, 8), make([]byte, 8), sig)) + + // Anchor: exactly 32 bytes, constant across runs. + anchor := []byte("swarm-test-anchor-deterministic!") + + sample, err := storer.MakeSampleUsingChunks([]swarm.Chunk{ch}, anchor) + if err != nil { + t.Fatal(err) + } + + if len(sample.Items) != 1 { + t.Fatalf("expected 1 sample item, got %d", len(sample.Items)) + } + + item := sample.Items[0] + + const ( + wantChunkAddr = "902406053a7a2f3a17f16097e1d0b4b6a4abeae6b84968f5503ae621f9522e16" + wantTransformedAddr = "9dee91d1ed794460474ffc942996bd713176731db4581a3c6470fe9862905a60" + ) + + if got := item.ChunkAddress.String(); got != wantChunkAddr { + t.Errorf("chunk address mismatch:\n got: %s\n want: %s", got, wantChunkAddr) + } + if got := item.TransformedAddress.String(); got != wantTransformedAddr { + t.Errorf("transformed address mismatch:\n got: %s\n want: %s", got, wantTransformedAddr) + } +} + func assertSampleNoErrors(t *testing.T, sample storer.Sample) { t.Helper() @@ -325,3 +382,99 @@ func assertSampleNoErrors(t *testing.T, sample storer.Sample) { t.Fatalf("got unexpected invalid stamps") } } + +// Benchmark results: +// goos: linux +// goarch: amd64 +// pkg: github.com/ethersphere/bee/v2/pkg/storer +// cpu: Intel(R) Core(TM) Ultra 7 165U +// BenchmarkCachePutter-14 473118 2149 ns/op 1184 B/op 24 allocs/op +// BenchmarkReservePutter-14 48109 29760 ns/op 12379 B/op 141 allocs/op +// BenchmarkReserveSample1k-14 100 12392598 ns/op 9364970 B/op 161383 allocs/op +// BenchmarkSampleHashing/chunks=1000-14 9 127425952 ns/op 32.14 MB/s 69386109 B/op 814005 allocs/op +// BenchmarkSampleHashing/chunks=10000-14 1 1241432669 ns/op 32.99 MB/s 693843032 B/op 8140005 allocs/op +// PASS +// ok github.com/ethersphere/bee/v2/pkg/storer 34.319s + +// BenchmarkReserveSample measures the end-to-end time of the ReserveSample +// method, including DB iteration, chunk loading, stamp validation, and sample +// assembly. +func BenchmarkReserveSample1k(b *testing.B) { + const chunkCountPerPO = 100 + const maxPO = 10 + + baseAddr := swarm.RandAddress(b) + opts := dbTestOps(baseAddr, 5000, nil, nil, time.Second) + opts.ValidStamp = func(ch swarm.Chunk) (swarm.Chunk, error) { return ch, nil } + + st, err := diskStorer(b, opts)() + if err != nil { + b.Fatal(err) + } + + timeVar := uint64(time.Now().UnixNano()) + + putter := st.ReservePutter() + for po := range maxPO { + for range chunkCountPerPO { + ch := chunk.GenerateValidRandomChunkAt(b, baseAddr, po).WithBatch(3, 2, false) + ch = ch.WithStamp(postagetesting.MustNewStampWithTimestamp(timeVar - 1)) + if err := putter.Put(context.Background(), ch); err != nil { + b.Fatal(err) + } + } + } + + var ( + radius uint8 = 5 + anchor = swarm.RandAddressAt(b, baseAddr, int(radius)).Bytes() + ) + + b.ResetTimer() + + for range b.N { + _, err := st.ReserveSample(context.TODO(), anchor, radius, timeVar, nil) + if err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkSampleHashing measures the time taken by MakeSampleUsingChunks to +// hash a fixed set of CAC chunks. +func BenchmarkSampleHashing(b *testing.B) { + anchor := []byte("swarm-test-anchor-deterministic!") + + // Shared zero-value stamp: its contents don't affect hash computation. + stamp := postage.NewStamp(make([]byte, 32), make([]byte, 8), make([]byte, 8), make([]byte, 65)) + + for _, count := range []int{1_000, 10_000} { + b.Run(fmt.Sprintf("chunks=%d", count), func(b *testing.B) { + // Build chunks once outside the measured loop. + // Content is derived deterministically from the chunk index so + // that every run produces the same set of chunk addresses. + chunks := make([]swarm.Chunk, count) + content := make([]byte, swarm.ChunkSize) + for i := range chunks { + for j := range content { + content[j] = byte(i + j) + } + ch, err := cac.New(content) + if err != nil { + b.Fatal(err) + } + chunks[i] = ch.WithStamp(stamp) + } + + // Report throughput so the output shows MB/s as well as ns/op. + b.SetBytes(int64(count) * swarm.ChunkSize) + b.ResetTimer() + + for range b.N { + if _, err := storer.MakeSampleUsingChunks(chunks, anchor); err != nil { + b.Fatal(err) + } + } + }) + } +} From c6357fdd9877d39d703685b022ca92c31c8b2967 Mon Sep 17 00:00:00 2001 From: Calin M Date: Thu, 26 Feb 2026 18:38:53 +0200 Subject: [PATCH 94/96] fix: fixes for topology oversaturated and block list streams tests (#5345) --- pkg/p2p/libp2p/connections_test.go | 14 +++++++------- pkg/p2p/libp2p/internal/handshake/handshake.go | 12 ++++++++++-- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/pkg/p2p/libp2p/connections_test.go b/pkg/p2p/libp2p/connections_test.go index 46ecca268e8..a31eb908f15 100644 --- a/pkg/p2p/libp2p/connections_test.go +++ b/pkg/p2p/libp2p/connections_test.go @@ -1070,10 +1070,9 @@ func TestTopologyOverSaturated(t *testing.T) { addr := serviceUnderlayAddress(t, s1) // s2 connects to s1, thus the notifier on s1 should be called on Connect - _, err := s2.Connect(ctx, addr) - if err == nil { - t.Fatal("expected connect to fail but it didn't") - } + // Connect might return nil if the handshake completes before the server processes the rejection (protocol race). + // We verify that the peer is eventually disconnected. + _, _ = s2.Connect(ctx, addr) expectPeers(t, s1) expectPeersEventually(t, s2) @@ -1174,9 +1173,10 @@ func TestWithBlocklistStreams(t *testing.T) { expectPeersEventually(t, s2) expectPeersEventually(t, s1) - if _, err := s2.Connect(ctx, s1_underlay); err == nil { - t.Fatal("expected error when connecting to blocklisted peer") - } + // s2 connects to s1, but because of blocklist it should fail + // Connect might return nil if the handshake completes before the server processes the blocklist (protocol race). + // We verify that the peer is eventually disconnected. + _, _ = s2.Connect(ctx, s1_underlay) expectPeersEventually(t, s2) expectPeersEventually(t, s1) diff --git a/pkg/p2p/libp2p/internal/handshake/handshake.go b/pkg/p2p/libp2p/internal/handshake/handshake.go index 63977652601..e88dc02ef2b 100644 --- a/pkg/p2p/libp2p/internal/handshake/handshake.go +++ b/pkg/p2p/libp2p/internal/handshake/handshake.go @@ -10,6 +10,7 @@ import ( "errors" "fmt" "slices" + "sync" "sync/atomic" "time" @@ -94,6 +95,7 @@ type Service struct { libp2pID libp2ppeer.ID metrics metrics picker p2p.Picker + mu sync.RWMutex hostAddresser Addresser } @@ -136,6 +138,8 @@ func New(signer crypto.Signer, advertisableAddresser AdvertisableAddressResolver } func (s *Service) SetPicker(n p2p.Picker) { + s.mu.Lock() + defer s.mu.Unlock() s.picker = n } @@ -351,8 +355,12 @@ func (s *Service) Handle(ctx context.Context, stream p2p.Stream, peerMultiaddrs overlay := swarm.NewAddress(ack.Address.Overlay) - if s.picker != nil { - if !s.picker.Pick(p2p.Peer{Address: overlay, FullNode: ack.FullNode}) { + s.mu.RLock() + picker := s.picker + s.mu.RUnlock() + + if picker != nil { + if !picker.Pick(p2p.Peer{Address: overlay, FullNode: ack.FullNode}) { return nil, ErrPicker } } From 253d867b6aac79e8aa693aff2c18ea443cb77bdc Mon Sep 17 00:00:00 2001 From: acud <12988138+acud@users.noreply.github.com> Date: Fri, 27 Feb 2026 06:29:21 -0600 Subject: [PATCH 95/96] test: fix soc flaky test (#5382) --- pkg/api/soc_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/api/soc_test.go b/pkg/api/soc_test.go index fb34eb82297..b1240708300 100644 --- a/pkg/api/soc_test.go +++ b/pkg/api/soc_test.go @@ -95,6 +95,12 @@ func TestSOC(t *testing.T) { }), ) + // Wait for the chanStorer drain goroutine to process the chunk and invoke + // the subscriber (which stores it in mockStorer) before issuing GETs. + if err := spinlock.Wait(2*time.Second, func() bool { return chanStore.Has(s.Address()) }); err != nil { + t.Fatal("timed out waiting for chunk to be stored:", err) + } + // try to fetch the same chunk t.Run("chunks fetch", func(t *testing.T) { rsrc := fmt.Sprintf("/chunks/%s", s.Address().String()) From ece6db4e270d246f94dd19e4918657ac3afa8e04 Mon Sep 17 00:00:00 2001 From: acud <12988138+acud@users.noreply.github.com> Date: Fri, 27 Feb 2026 08:54:55 -0600 Subject: [PATCH 96/96] ci: fix sonarqube warnings (#5385) --- .github/workflows/go.yml | 4 ++-- .github/workflows/release.yaml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 4bbfa5be9bd..f2d6523c4f4 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -78,7 +78,7 @@ jobs: with: configFile: commitlint.config.js - name: GolangCI-Lint - uses: golangci/golangci-lint-action@v8 + uses: golangci/golangci-lint-action@4afd733a84b1f43292c63897423277bb7f4313a9 # v8 with: skip-cache: false version: v2.5.0 @@ -108,7 +108,7 @@ jobs: - name: Test with code coverage run: make cover=1 test-ci - name: Upload coverage to Codecov - uses: codecov/codecov-action@v5 + uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5 with: token: ${{ secrets.CODECOV_TOKEN }} fail_ci_if_error: true diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 877e4845a6a..d100446245e 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -24,9 +24,9 @@ jobs: printf ${{ secrets.DOCKERHUB_PASSWORD }} | docker login --username ${{ secrets.DOCKERHUB_USERNAME }} --password-stdin printf ${{ secrets.QUAY_PASSWORD }} | docker login --username ${{ secrets.QUAY_USERNAME }} quay.io --password-stdin - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 - name: Import GPG key run: | echo "$GPG_PRIVATE_KEY" | gpg --import --passphrase "$GPG_PASSPHRASE" --batch --allow-secret-key-import @@ -37,7 +37,7 @@ jobs: run: | echo "BEE_API_VERSION=$(grep '^ version:' openapi/Swarm.yaml | awk '{print $2}')" >> $GITHUB_ENV - name: Run GoReleaser - uses: goreleaser/goreleaser-action@v6 + uses: goreleaser/goreleaser-action@e435ccd777264be153ace6237001ef4d979d3a7a # v6 with: version: '~> v2' args: release --clean --timeout 1h