From 472db74e51cccdb9cb9702af9e59f2b2ce3afdb8 Mon Sep 17 00:00:00 2001 From: javi11 Date: Wed, 11 Mar 2026 23:17:30 +0100 Subject: [PATCH 1/3] feat: add NZB FileName helper, SRR parser, and project README - Add File.FileName() method to extract filename from NZB subject lines - Add nzb/srr.go SRR file parser with nzb/srr_test.go tests - Add comprehensive README.md covering architecture, API, and usage examples Co-Authored-By: Claude Sonnet 4.6 --- README.md | 587 ++++++++++++++++++++++++++++++++++++++++++++++++ nzb/nzb.go | 17 ++ nzb/srr.go | 114 ++++++++++ nzb/srr_test.go | 161 +++++++++++++ 4 files changed, 879 insertions(+) create mode 100644 README.md create mode 100644 nzb/srr.go create mode 100644 nzb/srr_test.go diff --git a/README.md b/README.md new file mode 100644 index 0000000..31e2173 --- /dev/null +++ b/README.md @@ -0,0 +1,587 @@ +# nntppool + +A high-performance NNTP connection pool library for Go. It manages multiple NNTP provider connections with pipelining, automatic failover, backup providers, and yEnc decoding — designed for usenet download applications that need maximum throughput across many providers. + +## Table of Contents + +- [Key Features](#key-features) +- [Tech Stack](#tech-stack) +- [Prerequisites](#prerequisites) +- [Getting Started](#getting-started) +- [Architecture Overview](#architecture-overview) +- [API Reference](#api-reference) +- [Configuration Reference](#configuration-reference) +- [Testing](#testing) +- [Speed Test Tool](#speed-test-tool) +- [Troubleshooting](#troubleshooting) +- [Contributing](#contributing) + +## Key Features + +- Multi-provider connection pooling with configurable connection counts per provider +- NNTP command pipelining (configurable inflight requests per connection) +- Weighted round-robin and FIFO dispatch strategies across providers +- Automatic failover to backup providers on article-not-found (430) +- Same-host deduplication: a 430 from one account on a host skips other accounts on the same host +- Provider removal on permanent failure (502 service unavailable) +- yEnc decoding with CRC32 validation using the rapidyenc library +- UU encoding detection and handling +- Streaming body delivery to an `io.Writer` without buffering +- Priority channel for urgent requests that preempt normal queue +- Idle timeout for connection teardown under light load +- Per-provider stats (bytes consumed, missing articles, errors, ping RTT) +- Built-in speed test using NZB files + +## Tech Stack + +- **Language**: Go 1.25+ +- **Module**: `github.com/javi11/nntppool/v4` +- **Key dependency**: `github.com/mnightingale/rapidyenc` — native yEnc decoder +- **Metrics**: atomic counters, no external monitoring framework required +- **Test tooling**: standard `testing` package, `go-junit-report`, `golangci-lint` + +## Prerequisites + +- Go 1.25 or later (uses `go tool` for linting and testing utilities) +- Access to one or more NNTP provider accounts for integration testing against real servers +- No additional system packages required — rapidyenc is a pure-Go-compatible library + +## Getting Started + +### 1. Add the dependency + +```bash +go get github.com/javi11/nntppool/v4 +``` + +### 2. Basic usage — single provider + +```go +package main + +import ( + "context" + "fmt" + "log" + + "github.com/javi11/nntppool/v4" +) + +func main() { + ctx := context.Background() + + providers := []nntppool.Provider{ + { + Host: "news.example.com:563", + TLSConfig: &tls.Config{ServerName: "news.example.com"}, + Auth: nntppool.Auth{Username: "user", Password: "pass"}, + Connections: 20, + Inflight: 4, + }, + } + + client, err := nntppool.NewClient(ctx, providers) + if err != nil { + log.Fatal(err) + } + defer client.Close() + + // Fetch an article body (buffered) + body, err := client.Body(ctx, "some-message-id@example.com") + if err != nil { + log.Fatal(err) + } + fmt.Printf("Downloaded %d bytes (encoding: %v, CRC valid: %v)\n", + body.BytesDecoded, body.Encoding, body.CRCValid) +} +``` + +### 3. Multiple providers with backup + +```go +providers := []nntppool.Provider{ + { + Host: "news.provider1.com:563", + Auth: nntppool.Auth{Username: "u1", Password: "p1"}, + Connections: 30, + Inflight: 4, + }, + { + Host: "news.provider2.com:119", + Auth: nntppool.Auth{Username: "u2", Password: "p2"}, + Connections: 20, + Inflight: 2, + Backup: true, // only used when main providers return 430 + }, +} + +client, err := nntppool.NewClient(ctx, providers, + nntppool.WithDispatchStrategy(nntppool.DispatchRoundRobin), +) +``` + +### 4. Streaming body to a writer + +```go +var buf bytes.Buffer +body, err := client.BodyStream(ctx, "message-id@example.com", &buf) +if err != nil { + log.Fatal(err) +} +// body.Bytes is nil; buf holds the decoded bytes +// body.YEnc contains metadata (filename, size, part info) +fmt.Printf("File: %s, Part: %d/%d\n", body.YEnc.FileName, body.YEnc.Part, body.YEnc.Total) +``` + +### 5. Check article existence + +```go +stat, err := client.Stat(ctx, "message-id@example.com") +if errors.Is(err, nntppool.ErrArticleNotFound) { + fmt.Println("article missing") +} else if err != nil { + log.Fatal(err) +} +``` + +### 6. Fetch headers + +```go +head, err := client.Head(ctx, "message-id@example.com") +if err != nil { + log.Fatal(err) +} +fmt.Println(head.Headers["Subject"]) +``` + +### 7. Post an article + +```go +headers := nntppool.PostHeaders{ + From: "poster@example.com", + Subject: "Test post [1/1]", + Newsgroups: []string{"alt.test"}, + MessageID: "", +} +meta := rapidyenc.Meta{ + Filename: "test.bin", + Size: int64(len(data)), +} +result, err := client.PostYenc(ctx, headers, bytes.NewReader(data), meta) +if err != nil { + log.Fatal(err) +} +fmt.Printf("Posted: %d %s\n", result.StatusCode, result.Status) +``` + +--- + +## Architecture Overview + +### Connection Lifecycle + +Each provider is represented by a `providerGroup`, which owns: + +1. `**reqCh**` — buffered channel (capacity = `Connections`) for normal requests +2. `**prioCh**` — buffered channel for priority requests (`SendPriority`) +3. `**hotReqCh` / `hotPrioCh**` — unbuffered channels; only connected (hot) connections listen here + +When a request arrives at `Send()`: + +``` +Send() → doSendWithRetry() → round-robin/FIFO pick provider + → try hotReqCh (non-blocking, zero-copy if connection idle) + → fall back to reqCh (wakes a cold slot or queues behind in-flight) +``` + +Each connection slot runs as a goroutine pair: + +- `**writeLoop**`: reads from `pending`, writes NNTP commands to the TCP connection, handles POST two-phase handshake +- `**readerLoop**`: reads NNTP responses via `readBuffer.feedUntilDone()`, decodes yEnc/UU, delivers to `Response.RespCh` + +Both loops share a `pending` channel with capacity = `Inflight`, enforcing the pipelining depth. + +### Request Dispatch + +**Round-Robin (default)**: Uses dynamic weighted round-robin where weight = available inflight slots per provider. A provider with 10 connections contributes proportionally more than one with 2. The `nextIdx` atomic counter selects the start point using a cumulative-weight binary search. + +**FIFO**: Scans providers in order and sends to the first one with available capacity. Under light load this keeps traffic concentrated on the primary provider, minimizing unnecessary connections. + +### Failover and Retry Logic + +``` +Attempt main providers (round-robin or FIFO): + → 2xx: success, return response + → 430/423: article not found — try next provider (skip same-host duplicates) + → 502: provider permanently unavailable — remove from pool, continue + → connection error: try next provider + +If all mains return 430: attempt backup providers +If all providers exhausted: return last error +``` + +### Read Buffer + +The `readBuffer` starts at 128KB and grows in doublings up to 8MB when large yEnc segments require it. It caches the last `SetReadDeadline` call to avoid redundant syscalls on every read. + +### yEnc Decoding + +The parser in `reader.go` feeds raw NNTP response bytes incrementally through `NNTPResponse.Feed()`: + +1. Reads the status line (e.g., `222 0 body`) +2. Detects encoding format from the first data lines (`=ybegin`, `begin` , or UU line heuristics) +3. For yEnc: delegates to `rapidyenc.DecodeIncremental()` for SIMD-accelerated decoding +4. Accumulates CRC32 of decoded bytes, compares against `=yend pcrc32=` / `crc32=` +5. Fires the optional `onMeta` callback once `=ybegin`/`=ypart` headers are fully parsed + +### Hot/Cold Connection Model + +Connections are lazy: a cold slot only dials when a request actually arrives. Once connected, it registers on `hotReqCh` so subsequent requests can be dispatched without going through the buffered channel. A cold slot wakes when `reqCh` receives a request that overflows the current hot connections' inflight capacity. + +--- + +## API Reference + +### Creating a client + +```go +func NewClient(ctx context.Context, providers []Provider, opts ...ClientOption) (*Client, error) +``` + +Returns an error if providers is empty or contains duplicate names. Provider names default to `"host:port"` or a monotonic integer for factory-based providers. + +### Reading articles + +| Method | Description | +| ------------------------------------------ | ----------------------------------------- | +| `Body(ctx, messageID, onMeta...)` | Fetch and decode body into memory | +| `BodyStream(ctx, messageID, w, onMeta...)` | Decode and stream to `io.Writer` | +| `BodyAsync(ctx, messageID, w, onMeta...)` | Non-blocking; returns `<-chan BodyResult` | +| `BodyPriority(ctx, messageID, onMeta...)` | Like `Body` but uses the priority queue | +| `Head(ctx, messageID)` | Fetch RFC 5322 headers | +| `Stat(ctx, messageID)` | Check existence without transferring body | + +### Low-level send + +```go +// Send dispatches a raw NNTP command and returns a channel for the response. +func (c *Client) Send(ctx context.Context, payload []byte, bodyWriter io.Writer, onMeta ...func(YEncMeta)) <-chan Response + +// SendPriority is like Send but uses the priority queue. +func (c *Client) SendPriority(ctx context.Context, payload []byte, bodyWriter io.Writer, onMeta ...func(YEncMeta)) <-chan Response +``` + +Both return immediately; the caller receives exactly one `Response` from the channel. + +### Posting + +```go +func (c *Client) PostYenc(ctx context.Context, headers PostHeaders, body io.Reader, meta rapidyenc.Meta) (*PostResult, error) +``` + +Encodes the body as yEnc on the fly and sends using the two-phase POST protocol (sends `POST`, waits for `340`, then streams the article). Uses the same dispatch strategy as normal requests so concurrent POSTs are spread across providers. + +### Statistics + +```go +func (c *Client) Stats() ClientStats +``` + +Returns a snapshot of per-provider metrics (bytes consumed, missing articles, errors, ping RTT, active/max connections) and aggregate totals. + +### Lifecycle + +```go +func (c *Client) Close() error // cancel context, wait for goroutines +func (c *Client) NumProviders() int // number of active providers (main + backup) +``` + +### Key types + +```go +type ArticleBody struct { + MessageID string + Bytes []byte // nil when BodyStream was used + BytesDecoded int + BytesConsumed int // wire bytes (pre-decode) + Encoding ArticleEncoding // EncodingYEnc, EncodingUU, EncodingUnknown + YEnc YEncMeta // filename, filesize, part info + CRC uint32 + ExpectedCRC uint32 + CRCValid bool +} + +type YEncMeta struct { + FileName string + FileSize int64 + Part int64 + PartBegin int64 + PartSize int64 + Total int64 +} + +type PostHeaders struct { + From string + Subject string + Newsgroups []string + MessageID string + Extra map[string][]string // additional RFC 5322 headers +} +``` + +--- + +## Configuration Reference + +### Provider fields + +| Field | Type | Default | Description | +| ----------------- | --------------- | ------------ | -------------------------------------------------------------- | +| `Host` | `string` | — | `host:port`, e.g. `news.example.com:563` | +| `TLSConfig` | `*tls.Config` | nil (no TLS) | Pass a config to enable TLS | +| `Auth` | `Auth` | — | `Username` and `Password` for AUTHINFO | +| `Connections` | `int` | — | Number of connection slots for this provider | +| `Inflight` | `int` | 1 | Max pipelined requests per connection | +| `Factory` | `ConnFactory` | nil | Custom dialer, overrides `Host`/`TLSConfig` | +| `Backup` | `bool` | false | Only used when all main providers return 430 | +| `SkipPing` | `bool` | false | Skip the startup DATE ping (for servers that don't support it) | +| `IdleTimeout` | `time.Duration` | 0 (disabled) | Disconnect idle connections after this duration | +| `ThrottleRestore` | `time.Duration` | 30s | Wait before restoring throttled slots after 502 | +| `KeepAlive` | `time.Duration` | 30s | TCP keep-alive interval; negative disables | + +### Client options + +```go +// Set dispatch strategy (default: DispatchRoundRobin) +nntppool.WithDispatchStrategy(nntppool.DispatchFIFO) +nntppool.WithDispatchStrategy(nntppool.DispatchRoundRobin) +``` + +### Dispatch strategies + +| Strategy | Behavior | +| -------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `DispatchRoundRobin` | Weighted by available inflight capacity. Providers with more free slots receive proportionally more requests. | +| `DispatchFIFO` | First provider with available capacity gets the request. Under light load this keeps one provider "warm" and avoids unnecessary connections on others. | + +### Sentinel errors + +| Error | Meaning | +| ------------------------ | --------------------------------------------------------- | +| `ErrArticleNotFound` | NNTP 430 or 423 — article does not exist on this provider | +| `ErrPostingNotPermitted` | NNTP 440 — server does not allow posting | +| `ErrPostingFailed` | NNTP 441 — posting was rejected | +| `ErrAuthRequired` | NNTP 480 | +| `ErrAuthRejected` | NNTP 481 | +| `ErrServiceUnavailable` | NNTP 502 — provider removed from pool | +| `ErrCRCMismatch` | yEnc CRC32 check failed (body returned alongside error) | +| `ErrMaxConnections` | Server reported max connections reached during handshake | +| `ErrConnectionDied` | TCP connection closed unexpectedly | +| `ErrProtocolDesync` | Binary data received where a status line was expected | + +`ErrArticleNotFound` uses semantic category matching: both 430 and 423 satisfy `errors.Is(err, ErrArticleNotFound)`. + +--- + +## Testing + +### Running tests + +```bash +# All tests +go test ./... + +# With race detector (recommended before committing) +go test -race ./... + +# Specific package +go test ./nzb/... + +# Specific test +go test -run TestClient_SendRetryRoundRobin ./... + +# With verbose output +go test -v ./... + +# Coverage +go test -coverprofile=coverage.out ./... +go tool cover -html=coverage.out +``` + +### Benchmarks + +```bash +go test -bench=. -benchmem ./... + +# Specific benchmark +go test -bench=BenchmarkSend -benchmem -benchtime=5s ./... +``` + +Built-in benchmarks cover: + +- Equal-weight two-provider round-robin (3+3 connections) +- Weighted two-provider round-robin (5+1 connections) +- Single-provider baseline + +--- + +## Speed Test Tool + +`cmd/speedtest` is a command-line tool that measures download speed using an NZB file through the pool. + +### Build + +```bash +go build ./cmd/speedtest +``` + +### Usage — single provider (legacy flags) + +```bash +./speedtest \ + --host news.example.com:563 \ + --tls \ + --user myuser \ + --pass mypassword \ + --conns 20 \ + --inflight 4 +``` + +### Usage — multiple providers + +```bash +./speedtest \ + --provider "host=news.provider1.com:563,tls,user=u1,pass=p1,conns=20,inflight=4" \ + --provider "host=news.provider2.com:119,user=u2,pass=p2,conns=10,inflight=2,backup" +``` + +### Provider flag options + +| Option | Example | Description | +| ----------- | --------------------------- | ---------------------------------------------- | +| `host` | `host=news.example.com:563` | Server host and port (required) | +| `tls` | `tls` or `tls=true` | Enable TLS with SNI from host | +| `user` | `user=myuser` | NNTP username | +| `pass` | `pass=mypassword` | NNTP password | +| `conns` | `conns=20` | Number of connections (default: 10) | +| `inflight` | `inflight=4` | Pipelined requests per connection (default: 1) | +| `backup` | `backup` | Mark as backup provider | +| `idle` | `idle=30s` | Idle disconnect timeout | +| `throttle` | `throttle=30s` | Throttle restore duration | +| `keepalive` | `keepalive=60s` | TCP keep-alive interval | + +### Other flags + +| Flag | Default | Description | +| ----------------- | --------------------- | ------------------------------------- | +| `--nzb` | SABnzbd 10GB test NZB | Path or URL to NZB file | +| `--max-segments` | 0 (all) | Limit number of segments | +| `--provider-name` | (all) | Test only a specific provider by name | + +### Example output + +``` +Provider 1: news.example.com:563 (TLS: yes, conns: 20, inflight: 4, idle: none, keepalive: 1m0s, main) +Creating client with 20 connection slots across 1 provider(s)... +Ready (connections on demand). + +[ 15.3s] 450/1250 segs | 142.3 MB/s (avg 138.7 MB/s) | ETA 28s + +=== Speed Test Results === +Time: 45.123s +Segments: 1250 done, 0 missing, 0 errors +Wire: 1024.00 MB (22.70 MB/s) +Decoded: 981.44 MB (21.76 MB/s) +``` + +--- + +## Troubleshooting + +### Connection refused or timeout + +**Symptom:** `NewClient` hangs or connections fail immediately. + +**Check:** + +- Verify `host:port` is reachable: `nc -zv news.example.com 563` +- Confirm TLS settings match the port (563 = TLS, 119 = plain) +- For servers that reject the initial DATE ping, set `SkipPing: true` + +### Authentication rejected (481) + +**Symptom:** `ErrAuthRejected` on every request. + +**Check:** + +- Credentials are correct +- Some providers require the username in `user@domain.com` format +- Username/password must not contain commas if using the speedtest CLI `--provider` flag (use individual flags instead) + +### All articles return 430 + +**Symptom:** `ErrArticleNotFound` for known-good articles. + +**Check:** + +- The provider may not carry the newsgroup or the retention period has passed +- Add a backup provider that carries the content: set `Backup: true` +- Verify the message ID format (should include angle brackets in the `Send` payload but not in `Body`/`Head`/`Stat` calls — the API adds them automatically) + +### Max connections errors (502/400) + +**Symptom:** Connections are throttled immediately; `ErrMaxConnections` during connect. + +**Behaviour:** The pool automatically reduces active slots and restores them after `ThrottleRestore` (default 30 seconds). To adjust: + +```go +nntppool.Provider{ + ThrottleRestore: 60 * time.Second, +} +``` + +### Provider removed from pool + +**Symptom:** `NumProviders()` decreases; `ErrServiceUnavailable` returned. + +**Cause:** A connection returned 502 at the command level (not just during handshake). This signals permanent unavailability. The provider is removed from main or backup groups atomically. To restart, create a new `Client`. + +### CRC mismatch on decoded bodies + +**Symptom:** `ErrCRCMismatch` returned alongside a non-nil `*ArticleBody`. + +**Behaviour:** The body is returned even on CRC mismatch so callers can decide whether to discard or use the data. The `ArticleBody.CRCValid` field is `false` and `CRC != ExpectedCRC`. + +### Race conditions in tests + +**Symptom:** Test failures that only appear with `-race`. + +**Fix:** Ensure any shared state accessed from the mock server goroutine is protected by a mutex. See `TestClient_SendRetryRoundRobin` for the correct pattern. + +### Linter errors (`errcheck`) + +**Symptom:** `golangci-lint` fails with unchecked error returns. + +**Fix:** `io.PipeWriter.CloseWithError` and `io.PipeReader.Close` return values must be handled. Use the blank identifier explicitly: + +```go +defer func() { _ = pw.CloseWithError(err) }() +_ = pr.Close() +``` + +--- + +## Contributing + +In brief: + +1. Create a topic branch +2. Add tests for your change (aim for 100% coverage on new code) +3. Run `make check` — this runs generate, tidy, lint, and the race-detector test suite +4. Open a pull request + +The pre-commit hook (`make git-hooks`) runs the full `make` pipeline automatically. + +## License + +See [LICENSE](LICENSE). diff --git a/nzb/nzb.go b/nzb/nzb.go index f6dd7ba..2bf2990 100644 --- a/nzb/nzb.go +++ b/nzb/nzb.go @@ -4,6 +4,7 @@ import ( "encoding/xml" "fmt" "io" + "strings" ) type NZB struct { @@ -32,6 +33,22 @@ func Parse(r io.Reader) (*NZB, error) { return &n, nil } +// FileName extracts the filename from the NZB subject line. +// It returns the first token enclosed in double quotes that precedes " yEnc", +// or an empty string if the subject does not match that pattern. +func (f *File) FileName() string { + s := f.Subject + start := strings.Index(s, `"`) + if start == -1 { + return "" + } + end := strings.Index(s[start+1:], `"`) + if end == -1 { + return "" + } + return s[start+1 : start+1+end] +} + // AllSegments returns a flat list of all segments across all files. func (n *NZB) AllSegments() []Segment { var segs []Segment diff --git a/nzb/srr.go b/nzb/srr.go new file mode 100644 index 0000000..e5d38a2 --- /dev/null +++ b/nzb/srr.go @@ -0,0 +1,114 @@ +package nzb + +import ( + "encoding/binary" + "fmt" + "io" + "strings" +) + +const srrStoredFileType = 0x6A +const srrFlagAddSize = uint16(0x8000) + +// SRRFile returns the first File in the NZB whose FileName ends with ".srr" +// (case-insensitive), or nil if none. +func (n *NZB) SRRFile() *File { + for i := range n.Files { + if strings.HasSuffix(strings.ToLower(n.Files[i].FileName()), ".srr") { + return &n.Files[i] + } + } + return nil +} + +// ParseSRRFilenames parses an SRR binary and returns all original release +// filenames it encodes. It collects the filename of every stored file block +// (NFO, SFV, SRS, …) plus every filename listed in any embedded .sfv file. +// The combined, deduplicated list gives the full correct file list. +func ParseSRRFilenames(r io.Reader) ([]string, error) { + data, err := io.ReadAll(r) + if err != nil { + return nil, fmt.Errorf("srr read: %w", err) + } + if len(data) < 3 || data[0] != 0x69 || data[1] != 0x69 || data[2] != 0x69 { + return nil, fmt.Errorf("srr: invalid magic bytes") + } + + seen := make(map[string]bool) + var results []string + add := func(name string) { + if !seen[name] { + seen[name] = true + results = append(results, name) + } + } + + pos := 0 + for pos < len(data) { + if pos+7 > len(data) { + break + } + headType := data[pos+2] + headFlags := binary.LittleEndian.Uint16(data[pos+3:]) + headSize := int(binary.LittleEndian.Uint16(data[pos+5:])) + if headSize < 7 { + break + } + + var addSize uint32 + hasAddSize := headFlags&srrFlagAddSize != 0 + if hasAddSize { + if pos+11 > len(data) { + break + } + addSize = binary.LittleEndian.Uint32(data[pos+7:]) + } + + headerEnd := pos + headSize + if headerEnd > len(data) { + break + } + + if headType == srrStoredFileType { + bodyStart := pos + 7 + if hasAddSize { + bodyStart = pos + 11 + } + if bodyStart+2 <= headerEnd { + fnameLen := int(binary.LittleEndian.Uint16(data[bodyStart:])) + if bodyStart+2+fnameLen <= headerEnd { + fname := string(data[bodyStart+2 : bodyStart+2+fnameLen]) + add(fname) + if strings.HasSuffix(strings.ToLower(fname), ".sfv") && addSize > 0 { + sfvEnd := headerEnd + int(addSize) + if sfvEnd <= len(data) { + for _, n := range parseSFV(data[headerEnd:sfvEnd]) { + add(n) + } + } + } + } + } + } + + pos = headerEnd + int(addSize) + } + + return results, nil +} + +// parseSFV parses SFV file content and returns the filenames. +// Each non-comment line has the form: filename checksum +func parseSFV(data []byte) []string { + var names []string + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, ";") { + continue + } + if fields := strings.Fields(line); len(fields) > 0 { + names = append(names, fields[0]) + } + } + return names +} diff --git a/nzb/srr_test.go b/nzb/srr_test.go new file mode 100644 index 0000000..bd5d233 --- /dev/null +++ b/nzb/srr_test.go @@ -0,0 +1,161 @@ +package nzb + +import ( + "bytes" + "encoding/binary" + "strings" + "testing" +) + +func TestFileFileName(t *testing.T) { + tests := []struct { + subject string + want string + }{ + {`[01/66] - "nova.s44e02.720p-dhd.r00" yEnc (01/66)`, "nova.s44e02.720p-dhd.r00"}, + {`"simple.nfo" yEnc (1/1)`, "simple.nfo"}, + {`no quotes here`, ""}, + {`"unclosed`, ""}, + {`[1/1] - "release.srr" yEnc (1/1)`, "release.srr"}, + } + for _, tc := range tests { + f := File{Subject: tc.subject} + got := f.FileName() + if got != tc.want { + t.Errorf("FileName(%q) = %q, want %q", tc.subject, got, tc.want) + } + } +} + +func TestNZBSRRFile(t *testing.T) { + nzbXML := ` + + + + alt.binaries.test + abc@def + + + alt.binaries.test + ghi@jkl + +` + n, err := Parse(strings.NewReader(nzbXML)) + if err != nil { + t.Fatalf("Parse: %v", err) + } + f := n.SRRFile() + if f == nil { + t.Fatal("SRRFile() returned nil, want non-nil") + } + name := f.FileName() + if !strings.HasSuffix(strings.ToLower(name), ".srr") { + t.Errorf("SRRFile().FileName() = %q, want .srr suffix", name) + } +} + +func TestNZBSRRFileNone(t *testing.T) { + nzbXML := ` + + + alt.binaries.test + abc@def + +` + n, err := Parse(strings.NewReader(nzbXML)) + if err != nil { + t.Fatalf("Parse: %v", err) + } + if f := n.SRRFile(); f != nil { + t.Errorf("SRRFile() = %v, want nil", f) + } +} + +// buildSRRMarker returns the 7-byte SRR marker block (magic 0x69 0x69 0x69). +func buildSRRMarker() []byte { + // HEAD_CRC=0x6969, HEAD_TYPE=0x69, HEAD_FLAGS=0x0000, HEAD_SIZE=7 + return []byte{0x69, 0x69, 0x69, 0x00, 0x00, 0x07, 0x00} +} + +// buildStoredFileBlock builds a 0x6A stored-file block with the given +// filename and file content. +func buildStoredFileBlock(fname string, content []byte) []byte { + var buf bytes.Buffer + fnameBytes := []byte(fname) + // body = fname_len(2) + fname bytes + bodySize := 2 + len(fnameBytes) + // HEAD_SIZE includes base(7) + ADD_SIZE field(4) + body + headSize := uint16(7 + 4 + bodySize) + headFlags := uint16(0x8000) // has ADD_SIZE + addSize := uint32(len(content)) + + buf.Write([]byte{0x00, 0x00}) // HEAD_CRC (arbitrary for tests) + buf.WriteByte(0x6A) // HEAD_TYPE + _ = binary.Write(&buf, binary.LittleEndian, headFlags) + _ = binary.Write(&buf, binary.LittleEndian, headSize) + _ = binary.Write(&buf, binary.LittleEndian, addSize) + _ = binary.Write(&buf, binary.LittleEndian, uint16(len(fnameBytes))) + buf.Write(fnameBytes) + buf.Write(content) + return buf.Bytes() +} + +func TestParseSRRFilenames_Synthetic(t *testing.T) { + sfvContent := "; generated by test\nfile.r00 AABBCCDD\nfile.rar E1E2E3E4\n" + nfoContent := "some nfo text" + + var srrBuf bytes.Buffer + srrBuf.Write(buildSRRMarker()) + srrBuf.Write(buildStoredFileBlock("release.nfo", []byte(nfoContent))) + srrBuf.Write(buildStoredFileBlock("release.sfv", []byte(sfvContent))) + + names, err := ParseSRRFilenames(&srrBuf) + if err != nil { + t.Fatalf("ParseSRRFilenames: %v", err) + } + + want := map[string]bool{ + "release.nfo": true, + "release.sfv": true, + "file.r00": true, + "file.rar": true, + } + for _, name := range names { + delete(want, name) + } + if len(want) > 0 { + missing := make([]string, 0, len(want)) + for k := range want { + missing = append(missing, k) + } + t.Errorf("missing filenames: %v (got: %v)", missing, names) + } +} + +func TestParseSRRFilenames_InvalidMagic(t *testing.T) { + _, err := ParseSRRFilenames(bytes.NewReader([]byte{0x00, 0x01, 0x02, 0x03})) + if err == nil { + t.Error("expected error for invalid magic, got nil") + } +} + +func TestParseSRRFilenames_Dedup(t *testing.T) { + sfvContent := "file.r00 AABBCCDD\nfile.r00 AABBCCDD\n" // duplicate in SFV + var srrBuf bytes.Buffer + srrBuf.Write(buildSRRMarker()) + srrBuf.Write(buildStoredFileBlock("release.sfv", []byte(sfvContent))) + + names, err := ParseSRRFilenames(&srrBuf) + if err != nil { + t.Fatalf("ParseSRRFilenames: %v", err) + } + seen := make(map[string]int) + for _, n := range names { + seen[n]++ + } + for name, count := range seen { + if count > 1 { + t.Errorf("filename %q appears %d times, want 1", name, count) + } + } +} From 9336f398ae682cb114741b01d4ad49cc4cd6a231 Mon Sep 17 00:00:00 2001 From: Javier Date: Wed, 11 Mar 2026 23:18:27 +0100 Subject: [PATCH 2/3] Delete nzb/nzb.go --- nzb/nzb.go | 59 ------------------------------------------------------ 1 file changed, 59 deletions(-) delete mode 100644 nzb/nzb.go diff --git a/nzb/nzb.go b/nzb/nzb.go deleted file mode 100644 index 2bf2990..0000000 --- a/nzb/nzb.go +++ /dev/null @@ -1,59 +0,0 @@ -package nzb - -import ( - "encoding/xml" - "fmt" - "io" - "strings" -) - -type NZB struct { - Files []File `xml:"file"` -} - -type File struct { - Poster string `xml:"poster,attr"` - Date string `xml:"date,attr"` - Subject string `xml:"subject,attr"` - Groups []string `xml:"groups>group"` - Segments []Segment `xml:"segments>segment"` -} - -type Segment struct { - Bytes int `xml:"bytes,attr"` - Number int `xml:"number,attr"` - MessageID string `xml:",chardata"` -} - -func Parse(r io.Reader) (*NZB, error) { - var n NZB - if err := xml.NewDecoder(r).Decode(&n); err != nil { - return nil, fmt.Errorf("nzb parse: %w", err) - } - return &n, nil -} - -// FileName extracts the filename from the NZB subject line. -// It returns the first token enclosed in double quotes that precedes " yEnc", -// or an empty string if the subject does not match that pattern. -func (f *File) FileName() string { - s := f.Subject - start := strings.Index(s, `"`) - if start == -1 { - return "" - } - end := strings.Index(s[start+1:], `"`) - if end == -1 { - return "" - } - return s[start+1 : start+1+end] -} - -// AllSegments returns a flat list of all segments across all files. -func (n *NZB) AllSegments() []Segment { - var segs []Segment - for _, f := range n.Files { - segs = append(segs, f.Segments...) - } - return segs -} From 952aa4f4984e53d29951ee469d7b5b3d2916749e Mon Sep 17 00:00:00 2001 From: Javier Date: Wed, 11 Mar 2026 23:18:36 +0100 Subject: [PATCH 3/3] Delete nzb directory --- nzb/srr.go | 114 ---------------------------------- nzb/srr_test.go | 161 ------------------------------------------------ 2 files changed, 275 deletions(-) delete mode 100644 nzb/srr.go delete mode 100644 nzb/srr_test.go diff --git a/nzb/srr.go b/nzb/srr.go deleted file mode 100644 index e5d38a2..0000000 --- a/nzb/srr.go +++ /dev/null @@ -1,114 +0,0 @@ -package nzb - -import ( - "encoding/binary" - "fmt" - "io" - "strings" -) - -const srrStoredFileType = 0x6A -const srrFlagAddSize = uint16(0x8000) - -// SRRFile returns the first File in the NZB whose FileName ends with ".srr" -// (case-insensitive), or nil if none. -func (n *NZB) SRRFile() *File { - for i := range n.Files { - if strings.HasSuffix(strings.ToLower(n.Files[i].FileName()), ".srr") { - return &n.Files[i] - } - } - return nil -} - -// ParseSRRFilenames parses an SRR binary and returns all original release -// filenames it encodes. It collects the filename of every stored file block -// (NFO, SFV, SRS, …) plus every filename listed in any embedded .sfv file. -// The combined, deduplicated list gives the full correct file list. -func ParseSRRFilenames(r io.Reader) ([]string, error) { - data, err := io.ReadAll(r) - if err != nil { - return nil, fmt.Errorf("srr read: %w", err) - } - if len(data) < 3 || data[0] != 0x69 || data[1] != 0x69 || data[2] != 0x69 { - return nil, fmt.Errorf("srr: invalid magic bytes") - } - - seen := make(map[string]bool) - var results []string - add := func(name string) { - if !seen[name] { - seen[name] = true - results = append(results, name) - } - } - - pos := 0 - for pos < len(data) { - if pos+7 > len(data) { - break - } - headType := data[pos+2] - headFlags := binary.LittleEndian.Uint16(data[pos+3:]) - headSize := int(binary.LittleEndian.Uint16(data[pos+5:])) - if headSize < 7 { - break - } - - var addSize uint32 - hasAddSize := headFlags&srrFlagAddSize != 0 - if hasAddSize { - if pos+11 > len(data) { - break - } - addSize = binary.LittleEndian.Uint32(data[pos+7:]) - } - - headerEnd := pos + headSize - if headerEnd > len(data) { - break - } - - if headType == srrStoredFileType { - bodyStart := pos + 7 - if hasAddSize { - bodyStart = pos + 11 - } - if bodyStart+2 <= headerEnd { - fnameLen := int(binary.LittleEndian.Uint16(data[bodyStart:])) - if bodyStart+2+fnameLen <= headerEnd { - fname := string(data[bodyStart+2 : bodyStart+2+fnameLen]) - add(fname) - if strings.HasSuffix(strings.ToLower(fname), ".sfv") && addSize > 0 { - sfvEnd := headerEnd + int(addSize) - if sfvEnd <= len(data) { - for _, n := range parseSFV(data[headerEnd:sfvEnd]) { - add(n) - } - } - } - } - } - } - - pos = headerEnd + int(addSize) - } - - return results, nil -} - -// parseSFV parses SFV file content and returns the filenames. -// Each non-comment line has the form: filename checksum -func parseSFV(data []byte) []string { - var names []string - for _, line := range strings.Split(string(data), "\n") { - line = strings.TrimSpace(line) - if line == "" || strings.HasPrefix(line, ";") { - continue - } - if fields := strings.Fields(line); len(fields) > 0 { - names = append(names, fields[0]) - } - } - return names -} diff --git a/nzb/srr_test.go b/nzb/srr_test.go deleted file mode 100644 index bd5d233..0000000 --- a/nzb/srr_test.go +++ /dev/null @@ -1,161 +0,0 @@ -package nzb - -import ( - "bytes" - "encoding/binary" - "strings" - "testing" -) - -func TestFileFileName(t *testing.T) { - tests := []struct { - subject string - want string - }{ - {`[01/66] - "nova.s44e02.720p-dhd.r00" yEnc (01/66)`, "nova.s44e02.720p-dhd.r00"}, - {`"simple.nfo" yEnc (1/1)`, "simple.nfo"}, - {`no quotes here`, ""}, - {`"unclosed`, ""}, - {`[1/1] - "release.srr" yEnc (1/1)`, "release.srr"}, - } - for _, tc := range tests { - f := File{Subject: tc.subject} - got := f.FileName() - if got != tc.want { - t.Errorf("FileName(%q) = %q, want %q", tc.subject, got, tc.want) - } - } -} - -func TestNZBSRRFile(t *testing.T) { - nzbXML := ` - - - - alt.binaries.test - abc@def - - - alt.binaries.test - ghi@jkl - -` - n, err := Parse(strings.NewReader(nzbXML)) - if err != nil { - t.Fatalf("Parse: %v", err) - } - f := n.SRRFile() - if f == nil { - t.Fatal("SRRFile() returned nil, want non-nil") - } - name := f.FileName() - if !strings.HasSuffix(strings.ToLower(name), ".srr") { - t.Errorf("SRRFile().FileName() = %q, want .srr suffix", name) - } -} - -func TestNZBSRRFileNone(t *testing.T) { - nzbXML := ` - - - alt.binaries.test - abc@def - -` - n, err := Parse(strings.NewReader(nzbXML)) - if err != nil { - t.Fatalf("Parse: %v", err) - } - if f := n.SRRFile(); f != nil { - t.Errorf("SRRFile() = %v, want nil", f) - } -} - -// buildSRRMarker returns the 7-byte SRR marker block (magic 0x69 0x69 0x69). -func buildSRRMarker() []byte { - // HEAD_CRC=0x6969, HEAD_TYPE=0x69, HEAD_FLAGS=0x0000, HEAD_SIZE=7 - return []byte{0x69, 0x69, 0x69, 0x00, 0x00, 0x07, 0x00} -} - -// buildStoredFileBlock builds a 0x6A stored-file block with the given -// filename and file content. -func buildStoredFileBlock(fname string, content []byte) []byte { - var buf bytes.Buffer - fnameBytes := []byte(fname) - // body = fname_len(2) + fname bytes - bodySize := 2 + len(fnameBytes) - // HEAD_SIZE includes base(7) + ADD_SIZE field(4) + body - headSize := uint16(7 + 4 + bodySize) - headFlags := uint16(0x8000) // has ADD_SIZE - addSize := uint32(len(content)) - - buf.Write([]byte{0x00, 0x00}) // HEAD_CRC (arbitrary for tests) - buf.WriteByte(0x6A) // HEAD_TYPE - _ = binary.Write(&buf, binary.LittleEndian, headFlags) - _ = binary.Write(&buf, binary.LittleEndian, headSize) - _ = binary.Write(&buf, binary.LittleEndian, addSize) - _ = binary.Write(&buf, binary.LittleEndian, uint16(len(fnameBytes))) - buf.Write(fnameBytes) - buf.Write(content) - return buf.Bytes() -} - -func TestParseSRRFilenames_Synthetic(t *testing.T) { - sfvContent := "; generated by test\nfile.r00 AABBCCDD\nfile.rar E1E2E3E4\n" - nfoContent := "some nfo text" - - var srrBuf bytes.Buffer - srrBuf.Write(buildSRRMarker()) - srrBuf.Write(buildStoredFileBlock("release.nfo", []byte(nfoContent))) - srrBuf.Write(buildStoredFileBlock("release.sfv", []byte(sfvContent))) - - names, err := ParseSRRFilenames(&srrBuf) - if err != nil { - t.Fatalf("ParseSRRFilenames: %v", err) - } - - want := map[string]bool{ - "release.nfo": true, - "release.sfv": true, - "file.r00": true, - "file.rar": true, - } - for _, name := range names { - delete(want, name) - } - if len(want) > 0 { - missing := make([]string, 0, len(want)) - for k := range want { - missing = append(missing, k) - } - t.Errorf("missing filenames: %v (got: %v)", missing, names) - } -} - -func TestParseSRRFilenames_InvalidMagic(t *testing.T) { - _, err := ParseSRRFilenames(bytes.NewReader([]byte{0x00, 0x01, 0x02, 0x03})) - if err == nil { - t.Error("expected error for invalid magic, got nil") - } -} - -func TestParseSRRFilenames_Dedup(t *testing.T) { - sfvContent := "file.r00 AABBCCDD\nfile.r00 AABBCCDD\n" // duplicate in SFV - var srrBuf bytes.Buffer - srrBuf.Write(buildSRRMarker()) - srrBuf.Write(buildStoredFileBlock("release.sfv", []byte(sfvContent))) - - names, err := ParseSRRFilenames(&srrBuf) - if err != nil { - t.Fatalf("ParseSRRFilenames: %v", err) - } - seen := make(map[string]int) - for _, n := range names { - seen[n]++ - } - for name, count := range seen { - if count > 1 { - t.Errorf("filename %q appears %d times, want 1", name, count) - } - } -}