panic on read

[sduchesneau]

saw on a substreams tier2 reading zstd-encoded data

panic: runtime error: slice bounds out of range [131072:0]

goroutine 23067 [running]:
github.com/klauspost/compress/zstd.(*Decoder).Read(0xc000c76200, {0xc03133131d?, 0x1c?, 0x20000?})
	/go/pkg/mod/github.com/klauspost/compress@v1.17.11/zstd/decoder.go:134 +0x1b9
github.com/klauspost/compress/zstd.closeWrapper.Read({0x7f6770301038?}, {0xc03133131d?, 0xc000d36008?, 0xc000d36008?})
	/go/pkg/mod/github.com/klauspost/compress@v1.17.11/zstd/decoder.go:616 +0x18
github.com/streamingfast/dstore.(*callbackReadCloser).Read(0xc00083fd10, {0xc03133131d?, 0xc0019d6540?, 0x4e83e9?})
	/go/pkg/mod/github.com/streamingfast/dstore@v0.1.1-0.20250609173504-95368d3441ee/callback.go:33 +0x25
io.ReadAtLeast({0x7f672867a9b0, 0xc00083fd10}, {0xc0312ac000, 0x214056, 0x214056}, 0x214056)
	/usr/local/go/src/io/io.go:335 +0x91
io.ReadFull(...)
	/usr/local/go/src/io/io.go:354
github.com/streamingfast/substreams/storage/execout/streamproto.ReadNextItem({0x7f672867a9b0, 0xc00083fd10})
	/go/pkg/mod/github.com/streamingfast/substreams@v1.15.9-0.20250616133405-432e58897ceb/storage/execout/streamproto/protostream.go:77 +0x1be
github.com/streamingfast/substreams/storage/execout.(*fileReader).ReadNext(0xc0038d77a0)
	/go/pkg/mod/github.com/streamingfast/substreams@v1.15.9-0.20250616133405-432e58897ceb/storage/execout/file.go:304 +0x65
github.com/streamingfast/substreams/storage/execout.(*ClockDistributor).Next(0xc0019d6b90, {0x460f380?, 0xc02ee8a360?})
	/go/pkg/mod/github.com/streamingfast/substreams@v1.15.9-0.20250616133405-432e58897ceb/storage/execout/file.go:106 +0x20a
github.com/streamingfast/substreams/service.(*Tier2Service).processRange.(*ClockDistributor).Iter.func4(...)
	/go/pkg/mod/github.com/streamingfast/substreams@v1.15.9-0.20250616133405-432e58897ceb/storage/execout/file.go:137
github.com/streamingfast/substreams/service.(*Tier2Service).processRange(0xc000c7b0e0, {0x460f380, 0xc0008349f0}, 0xc0023ca5b0, 0xc000dcfd00)
	/go/pkg/mod/github.com/streamingfast/substreams@v1.15.9-0.20250616133405-432e58897ceb/service/tier2.go:500 +0x15e7
github.com/streamingfast/substreams/service.(*Tier2Service).ProcessRange(0xc000c7b0e0, 0xc0023ca5b0, {0x461b3c0, 0xc026d82030})
	/go/pkg/mod/github.com/streamingfast/substreams@v1.15.9-0.20250616133405-432e58897ceb/service/tier2.go:319 +0x269a
github.com/streamingfast/substreams/pb/sf/substreams/intern/v2._Substreams_ProcessRange_Handler({0x3d9d200, 0xc000c7b0e0}, {0x4618e38, 0xc024c48e00})
	/go/pkg/mod/github.com/streamingfast/substreams@v1.15.9-0.20250616133405-432e58897ceb/pb/sf/substreams/intern/v2/service_grpc.pb.go:101 +0x110
google.golang.org/grpc/xds.xdsStreamInterceptor({0x3d9d200, 0xc000c7b0e0}, {0x4618e38, 0xc024c48e00}, 0x654341ca92ddc77c?, 0x41d5230)
	/go/pkg/mod/google.golang.org/grpc@v1.72.0/xds/server.go:322 +0x5f
google.golang.org/grpc.getChainStreamHandler.func1({0x3d9d200, 0xc000c7b0e0}, {0x4618e38, 0xc024c48e00})
	/go/pkg/mod/google.golang.org/grpc@v1.72.0/server.go:1548 +0xa8
github.com/streamingfast/substreams/service.ListenTier2.StreamAuthChecker.func5({0x3d9d200, 0xc000c7b0e0}, {0x4618970, 0xc018d75f00}, 0xc013057d28, 0xc018d75cc0)
	/go/pkg/mod/github.com/streamingfast/dauth@v0.0.0-20250130223258-c615a033a660/middleware/grpc/middleware.go:32 +0xfd
github.com/streamingfast/dgrpc/server/discovery-service/traffic-director.NewServer.WithStreamServerChain.ChainStreamServer.func7.1.1({0x3d9d200?, 0xc000c7b0e0?}, {0x4618970?, 0xc018d75f00?})
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/chain.go:49 +0x34
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x3d9d200, 0xc000c7b0e0}, {0x46191f0, 0xc024c48d00}, 0xc013057d28, 0xc024c48c00)
	/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.60.0/interceptor.go:436 +0x5b8
github.com/streamingfast/dgrpc/server/discovery-service/traffic-director.NewServer.WithStreamServerChain.ChainStreamServer.func7.1.1({0x3d9d200?, 0xc000c7b0e0?}, {0x46191f0?, 0xc024c48d00?})
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/chain.go:49 +0x34
github.com/streamingfast/dgrpc/server/discovery-service/traffic-director.NewServer.SetupLoggingInterceptors.func2({0x3d9d200, 0xc000c7b0e0}, {0x4618970, 0xc018d75d40}, 0x3dedbe0?, 0xc024c48c20)
	/go/pkg/mod/github.com/streamingfast/dgrpc@v0.0.0-20250423172640-223250ed2391/server/tracelog/tracelog.go:36 +0x9f
github.com/streamingfast/dgrpc/server/discovery-service/traffic-director.NewServer.WithStreamServerChain.ChainStreamServer.func7.1.1({0x3d9d200?, 0xc000c7b0e0?}, {0x4618970?, 0xc018d75d40?})
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/chain.go:49 +0x34
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x3d9d200, 0xc000c7b0e0}, {0x4619358, 0xc013057dd0}, 0xc013057d28, 0xc024c48c40)
	/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.60.0/interceptor.go:436 +0x5b8
github.com/streamingfast/dgrpc/server/discovery-service/traffic-director.NewServer.WithStreamServerChain.ChainStreamServer.func7.1.1({0x3d9d200?, 0xc000c7b0e0?}, {0x4619358?, 0xc013057dd0?})
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/chain.go:49 +0x34
github.com/grpc-ecosystem/go-grpc-prometheus.init.(*ServerMetrics).StreamServerInterceptor.func4({0x3d9d200, 0xc000c7b0e0}, {0x4617da0, 0xc000ab5a40}, 0xc013057d28, 0xc024c48c60)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xd2
github.com/streamingfast/dgrpc/server/discovery-service/traffic-director.NewServer.WithStreamServerChain.ChainStreamServer.func7.1.1({0x3d9d200?, 0xc000c7b0e0?}, {0x4617da0?, 0xc000ab5a40?})
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/chain.go:49 +0x34
github.com/streamingfast/dgrpc/server/discovery-service/traffic-director.NewServer.WithStreamServerChain.ChainStreamServer.func7({0x3d9d200, 0xc000c7b0e0}, {0x4617da0, 0xc000ab5a40}, 0xc013057d28?, 0x41d5230?)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/chain.go:58 +0xb4
google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x3d9d200, 0xc000c7b0e0}, {0x4617da0, 0xc000ab5a40}, 0xc013057d28, 0xc0156cbe60?)
	/go/pkg/mod/google.golang.org/grpc@v1.72.0/server.go:1539 +0x7c
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c0f200, {0x460f380, 0xc018feba10}, 0xc003a00120, 0xc0008a97d0, 0x640d4c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.72.0/server.go:1702 +0x1231
google.golang.org/grpc.(*Server).handleStream(0xc000c0f200, {0x460fcf8, 0xc000c784e0}, 0xc003a00120)
	/go/pkg/mod/google.golang.org/grpc@v1.72.0/server.go:1819 +0xb47
google.golang.org/grpc.(*Server).serveStreams.func2.1()
	/go/pkg/mod/google.golang.org/grpc@v1.72.0/server.go:1035 +0x7f
created by google.golang.org/grpc.(*Server).serveStreams.func2 in goroutine 300
	/go/pkg/mod/google.golang.org/grpc@v1.72.0/server.go:1046 +0x11d

[sduchesneau]

in zstd/decoder.go:

// Read bytes from the decompressed stream into p.
// Returns the number of bytes written and any error that occurred.
// When the stream is done, io.EOF will be returned.
func (d *Decoder) Read(p []byte) (int, error) {
	var n int
	for {
		if len(d.current.b) > 0 {
			filled := copy(p, d.current.b)
			p = p[filled:]
			d.current.b = d.current.b[filled:] // <--- this line panics
			n += filled
		}

I don't see how to reproduce a panic there, since 'filled' cannot be > the size of d.current.b
and the size of p is determined by io.ReadAll() which should be pretty battle-tested

copy(dst, src) should always returns the number of elements copied, which will be the minimum of len(src) and len(dst)

How can src[len(src)] be out of bounds ?
a byte slice cannot be shrunk in go...
nothin can change it under the hood in another thread AFAIK

mysterious!