From bdfe1cbaa9648ff6aaa45b2738a348967c91fd2f Mon Sep 17 00:00:00 2001 From: Max Mouchet Date: Thu, 22 Jan 2026 21:23:03 +0100 Subject: [PATCH 1/5] draft --- go.mod | 5 ++- go.sum | 12 ++++-- lib/cmd_export.go | 107 +++++++++++++++++++++++++++++++++------------- lib/writer_tsv.go | 14 +++--- 4 files changed, 98 insertions(+), 40 deletions(-) diff --git a/go.mod b/go.mod index 4a45cec..729edec 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/ipinfo/mmdbctl -go 1.20 +go 1.24.0 require ( github.com/edsrzf/mmap-go v1.1.0 @@ -8,6 +8,7 @@ require ( github.com/ipinfo/cli v0.0.0-20240814004006-a9ca4b1d939d github.com/maxmind/mmdbwriter v1.0.1-0.20231024181307-469cd9b959b4 github.com/oschwald/maxminddb-golang v1.12.0 + github.com/oschwald/maxminddb-golang/v2 v2.1.1 github.com/spf13/pflag v1.0.5 ) @@ -19,5 +20,5 @@ require ( github.com/posener/script v1.2.0 // indirect go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 // indirect - golang.org/x/sys v0.15.0 // indirect + golang.org/x/sys v0.38.0 // indirect ) diff --git a/go.sum b/go.sum index c3bb60c..084928c 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,5 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ= github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= @@ -19,18 +20,23 @@ github.com/maxmind/mmdbwriter v1.0.1-0.20231024181307-469cd9b959b4 h1:rF6qXloekm github.com/maxmind/mmdbwriter v1.0.1-0.20231024181307-469cd9b959b4/go.mod h1:6F/4tSDsJ8Y9UFVnehdZEIS220Uz62E7lbo8ZS0DehI= github.com/oschwald/maxminddb-golang v1.12.0 h1:9FnTOD0YOhP7DGxGsq4glzpGy5+w7pq50AS6wALUMYs= github.com/oschwald/maxminddb-golang v1.12.0/go.mod h1:q0Nob5lTCqyQ8WT6FYgS1L7PXKVVbgiymefNwIjPzgY= +github.com/oschwald/maxminddb-golang/v2 v2.1.1 h1:lA8FH0oOrM4u7mLvowq8IT6a3Q/qEnqRzLQn9eH5ojc= +github.com/oschwald/maxminddb-golang/v2 v2.1.1/go.mod h1:PLdx6PR+siSIoXqqy7C7r3SB3KZnhxWr1Dp6g0Hacl8= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/script v1.2.0 h1:DrZz0qFT8lCLkYNi1PleLDANFnKxJ2VmlNPJbAkVLsE= github.com/posener/script v1.2.0/go.mod h1:s4sVvRXtdc/1aK6otTSeW2BVXndO8MsoOVUwK74zcg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= go4.org/netipx v0.0.0-20231129151722-fdeea329fbba h1:0b9z3AuHCjxk0x/opv64kcgZLBseWJUpBw5I82+2U4M= go4.org/netipx v0.0.0-20231129151722-fdeea329fbba/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y= golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 h1:qCEDpW1G+vcj3Y7Fy52pEM1AWm3abj8WimGYejI3SC4= golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/lib/cmd_export.go b/lib/cmd_export.go index ad7c318..e228896 100644 --- a/lib/cmd_export.go +++ b/lib/cmd_export.go @@ -1,6 +1,7 @@ package lib import ( + "bufio" "encoding/csv" "encoding/json" "errors" @@ -8,7 +9,7 @@ import ( "os" "strings" - "github.com/oschwald/maxminddb-golang" + maxminddb "github.com/oschwald/maxminddb-golang/v2" "github.com/spf13/pflag" ) @@ -112,20 +113,38 @@ func CmdExport(f CmdExportFlags, args []string, printHelp func()) error { tsvwr := NewTsvWriter(outFile) wr = tsvwr } - record := make(map[string]interface{}) - networks := db.Networks(maxminddb.SkipAliasedNetworks) - for networks.Next() { - subnet, err := networks.Network(&record) - if err != nil { - return fmt.Errorf("failed to get record for next subnet: %w", err) + + // Cache for decoded and serialized records, keyed by data offset. + // Many networks can point to the same data record in MMDB files. + cache := make(map[uintptr]map[string]string) + var hdrKeys []string + + for result := range db.Networks() { + if err := result.Err(); err != nil { + return fmt.Errorf("failed networks traversal: %w", err) + } + + offset := result.Offset() + prefix := result.Prefix() + + var recordStr map[string]string + if cached, ok := cache[offset]; ok { + recordStr = cached + } else { + // Cache miss: decode and serialize. + record := make(map[string]any) + if err := result.Decode(&record); err != nil { + return fmt.Errorf("failed to decode record: %w", err) + } + recordStr = mapInterfaceToStr(record) + cache[offset] = recordStr } - recordStr := mapInterfaceToStr(record) if !hdrWritten { hdrWritten = true - + hdrKeys = sortedMapKeys(recordStr) if !f.NoHdr { - hdr := append([]string{"range"}, sortedMapKeys(recordStr)...) + hdr := append([]string{"range"}, hdrKeys...) if err := wr.Write(hdr); err != nil { return fmt.Errorf( "failed to write header %v: %w", @@ -135,10 +154,13 @@ func CmdExport(f CmdExportFlags, args []string, printHelp func()) error { } } - line := append( - []string{subnet.String()}, - sortedMapValsByKeys(recordStr)..., - ) + // Build values in header key order, using empty string for missing keys. + vals := make([]string, len(hdrKeys)) + for i, k := range hdrKeys { + vals[i] = recordStr[k] // Returns "" if key doesn't exist + } + + line := append([]string{prefix.String()}, vals...) if err := wr.Write(line); err != nil { return fmt.Errorf("failed to write line %v: %w", line, err) } @@ -147,25 +169,50 @@ func CmdExport(f CmdExportFlags, args []string, printHelp func()) error { if err := wr.Error(); err != nil { return fmt.Errorf("writer had failure: %w", err) } - if err := networks.Err(); err != nil { - return fmt.Errorf("failed networks traversal: %w", err) - } } else if f.Format == "json" { - networks := db.Networks(maxminddb.SkipAliasedNetworks) - enc := json.NewEncoder(outFile) - for networks.Next() { - record := make(map[string]interface{}) - - subnet, err := networks.Network(&record) - if err != nil { - return fmt.Errorf("failed to get record for next subnet: %w", err) + // Cache for JSON-encoded records (without "range" field), keyed by data offset. + cache := make(map[uintptr][]byte) + bw := bufio.NewWriter(outFile) + + for result := range db.Networks() { + if err := result.Err(); err != nil { + return fmt.Errorf("failed networks traversal: %w", err) } - record["range"] = subnet.String() - enc.Encode(record) - } - if err := networks.Err(); err != nil { - return fmt.Errorf("failed networks traversal: %w", err) + + offset := result.Offset() + prefix := result.Prefix() + + var jsonSuffix []byte + if cached, ok := cache[offset]; ok { + jsonSuffix = cached + } else { + // Cache miss: decode and encode to JSON. + record := make(map[string]any) + if err := result.Decode(&record); err != nil { + return fmt.Errorf("failed to decode record: %w", err) + } + + encoded, err := json.Marshal(record) + if err != nil { + return fmt.Errorf("failed to encode record: %w", err) + } + // Cache everything after the opening '{'. + jsonSuffix = encoded[1:] + cache[offset] = jsonSuffix + } + + // Write: {"range":"",...}\n + // jsonSuffix is either "}" (empty record) or `"key":val,...}` + bw.WriteString(`{"range":"`) + bw.WriteString(prefix.String()) + bw.WriteString(`"`) + if len(jsonSuffix) > 1 { // More than just "}" + bw.WriteByte(',') + } + bw.Write(jsonSuffix) + bw.WriteByte('\n') } + bw.Flush() } return nil } diff --git a/lib/writer_tsv.go b/lib/writer_tsv.go index 3b3e718..11d29b4 100644 --- a/lib/writer_tsv.go +++ b/lib/writer_tsv.go @@ -1,27 +1,31 @@ package lib import ( - "fmt" + "bufio" "io" "strings" ) type TsvWriter struct { - w io.Writer + bw *bufio.Writer } func NewTsvWriter(w io.Writer) *TsvWriter { return &TsvWriter{ - w: w, + bw: bufio.NewWriter(w), } } func (w *TsvWriter) Write(record []string) error { - _, err := fmt.Fprintln(w.w, strings.Join(record, "\t")) - return err + _, err := w.bw.WriteString(strings.Join(record, "\t")) + if err != nil { + return err + } + return w.bw.WriteByte('\n') } func (w *TsvWriter) Flush() { + w.bw.Flush() } func (w *TsvWriter) Error() error { From ea25aad8d3f660619062c2572fa1028be16cd02c Mon Sep 17 00:00:00 2001 From: Max Mouchet Date: Thu, 22 Jan 2026 21:29:31 +0100 Subject: [PATCH 2/5] go: upgrade to maxminddb-golang/v2 --- go.mod | 7 ++++--- go.sum | 12 +++++++++--- lib/cmd_diff.go | 34 +++++++++++++--------------------- lib/cmd_export.go | 26 ++++++++------------------ lib/cmd_import_test.go | 10 +++++----- lib/cmd_metadata.go | 2 +- lib/cmd_read.go | 15 +++++++++++++-- lib/cmd_verify.go | 2 +- 8 files changed, 54 insertions(+), 54 deletions(-) diff --git a/go.mod b/go.mod index 4a45cec..6072399 100644 --- a/go.mod +++ b/go.mod @@ -1,13 +1,13 @@ module github.com/ipinfo/mmdbctl -go 1.20 +go 1.24.0 require ( github.com/edsrzf/mmap-go v1.1.0 github.com/fatih/color v1.16.0 github.com/ipinfo/cli v0.0.0-20240814004006-a9ca4b1d939d github.com/maxmind/mmdbwriter v1.0.1-0.20231024181307-469cd9b959b4 - github.com/oschwald/maxminddb-golang v1.12.0 + github.com/oschwald/maxminddb-golang/v2 v2.0.0 github.com/spf13/pflag v1.0.5 ) @@ -16,8 +16,9 @@ require ( github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/oschwald/maxminddb-golang v1.12.0 // indirect github.com/posener/script v1.2.0 // indirect go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 // indirect - golang.org/x/sys v0.15.0 // indirect + golang.org/x/sys v0.37.0 // indirect ) diff --git a/go.sum b/go.sum index c3bb60c..7c53025 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,5 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ= github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= @@ -19,18 +20,23 @@ github.com/maxmind/mmdbwriter v1.0.1-0.20231024181307-469cd9b959b4 h1:rF6qXloekm github.com/maxmind/mmdbwriter v1.0.1-0.20231024181307-469cd9b959b4/go.mod h1:6F/4tSDsJ8Y9UFVnehdZEIS220Uz62E7lbo8ZS0DehI= github.com/oschwald/maxminddb-golang v1.12.0 h1:9FnTOD0YOhP7DGxGsq4glzpGy5+w7pq50AS6wALUMYs= github.com/oschwald/maxminddb-golang v1.12.0/go.mod h1:q0Nob5lTCqyQ8WT6FYgS1L7PXKVVbgiymefNwIjPzgY= +github.com/oschwald/maxminddb-golang/v2 v2.0.0 h1:Gyljxck1kHbBxDgLM++NfDWBqvu1pWWfT8XbosSo0bo= +github.com/oschwald/maxminddb-golang/v2 v2.0.0/go.mod h1:gG4V88LsawPEqtbL1Veh1WRh+nVSYwXzJ1P5Fcn77g0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/script v1.2.0 h1:DrZz0qFT8lCLkYNi1PleLDANFnKxJ2VmlNPJbAkVLsE= github.com/posener/script v1.2.0/go.mod h1:s4sVvRXtdc/1aK6otTSeW2BVXndO8MsoOVUwK74zcg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= go4.org/netipx v0.0.0-20231129151722-fdeea329fbba h1:0b9z3AuHCjxk0x/opv64kcgZLBseWJUpBw5I82+2U4M= go4.org/netipx v0.0.0-20231129151722-fdeea329fbba/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y= golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 h1:qCEDpW1G+vcj3Y7Fy52pEM1AWm3abj8WimGYejI3SC4= golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= +golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/lib/cmd_diff.go b/lib/cmd_diff.go index 76858be..071facb 100644 --- a/lib/cmd_diff.go +++ b/lib/cmd_diff.go @@ -1,13 +1,12 @@ package lib import ( - "bytes" "errors" "fmt" - "net" + "net/netip" "reflect" - "github.com/oschwald/maxminddb-golang" + "github.com/oschwald/maxminddb-golang/v2" "github.com/spf13/pflag" ) @@ -51,33 +50,32 @@ func doDiff( newDbStr string, oldDb *maxminddb.Reader, oldDbStr string, -) (map[interface{}]*net.IPNet, map[interface{}]cmdDiffRecord, error) { - modifiedSubnets := map[interface{}]*net.IPNet{} - modifiedRecords := map[interface{}]cmdDiffRecord{} - networksA := newDb.Networks(maxminddb.SkipAliasedNetworks) - for networksA.Next() { +) (map[netip.Prefix]netip.Prefix, map[netip.Prefix]cmdDiffRecord, error) { + modifiedSubnets := map[netip.Prefix]netip.Prefix{} + modifiedRecords := map[netip.Prefix]cmdDiffRecord{} + for result := range newDb.Networks() { var recordA interface{} var recordB interface{} - subnetA, err := networksA.Network(&recordA) - if err != nil { + if err := result.Decode(&recordA); err != nil { return nil, nil, fmt.Errorf( "failed to get record for subnet from %v: %w", newDbStr, err, ) } + subnetA := result.Prefix() - subnetB, _, err := oldDb.LookupNetwork(subnetA.IP, &recordB) - if err != nil { + lookupResult := oldDb.Lookup(subnetA.Addr()) + if err := lookupResult.Decode(&recordB); err != nil { return nil, nil, fmt.Errorf( "failed to get record for IP %v from %v: %w", - subnetA.IP, oldDbStr, err, + subnetA.Addr(), oldDbStr, err, ) } + subnetB := lookupResult.Prefix() // unequal subnets? - if bytes.Compare(subnetA.IP, subnetB.IP) != 0 || - bytes.Compare(subnetA.Mask, subnetB.Mask) != 0 { + if subnetA != subnetB { modifiedSubnets[subnetA] = subnetB continue } @@ -90,12 +88,6 @@ func doDiff( } } } - if networksA.Err() != nil { - return nil, nil, fmt.Errorf( - "failed traversing networks of %v: %w", - newDbStr, networksA.Err(), - ) - } return modifiedSubnets, modifiedRecords, nil } diff --git a/lib/cmd_export.go b/lib/cmd_export.go index ad7c318..0a29d0c 100644 --- a/lib/cmd_export.go +++ b/lib/cmd_export.go @@ -8,7 +8,7 @@ import ( "os" "strings" - "github.com/oschwald/maxminddb-golang" + "github.com/oschwald/maxminddb-golang/v2" "github.com/spf13/pflag" ) @@ -112,13 +112,12 @@ func CmdExport(f CmdExportFlags, args []string, printHelp func()) error { tsvwr := NewTsvWriter(outFile) wr = tsvwr } - record := make(map[string]interface{}) - networks := db.Networks(maxminddb.SkipAliasedNetworks) - for networks.Next() { - subnet, err := networks.Network(&record) - if err != nil { + for result := range db.Networks() { + record := make(map[string]interface{}) + if err := result.Decode(&record); err != nil { return fmt.Errorf("failed to get record for next subnet: %w", err) } + subnet := result.Prefix() recordStr := mapInterfaceToStr(record) if !hdrWritten { @@ -147,25 +146,16 @@ func CmdExport(f CmdExportFlags, args []string, printHelp func()) error { if err := wr.Error(); err != nil { return fmt.Errorf("writer had failure: %w", err) } - if err := networks.Err(); err != nil { - return fmt.Errorf("failed networks traversal: %w", err) - } } else if f.Format == "json" { - networks := db.Networks(maxminddb.SkipAliasedNetworks) enc := json.NewEncoder(outFile) - for networks.Next() { + for result := range db.Networks() { record := make(map[string]interface{}) - - subnet, err := networks.Network(&record) - if err != nil { + if err := result.Decode(&record); err != nil { return fmt.Errorf("failed to get record for next subnet: %w", err) } - record["range"] = subnet.String() + record["range"] = result.Prefix().String() enc.Encode(record) } - if err := networks.Err(); err != nil { - return fmt.Errorf("failed networks traversal: %w", err) - } } return nil } diff --git a/lib/cmd_import_test.go b/lib/cmd_import_test.go index 5e51cd8..8da6a76 100644 --- a/lib/cmd_import_test.go +++ b/lib/cmd_import_test.go @@ -1,13 +1,13 @@ package lib import ( - "net" + "net/netip" "os" "path/filepath" "strings" "testing" - "github.com/oschwald/maxminddb-golang" + "github.com/oschwald/maxminddb-golang/v2" ) // verifyMMDBContent is a test helper that verifies MMDB file contains expected entries @@ -29,14 +29,14 @@ func verifyMMDBContent(t *testing.T, mmdbPath string, testCases []struct { defer db.Close() for _, tc := range testCases { - ip := net.ParseIP(tc.ip) - if ip == nil { + addr, err := netip.ParseAddr(tc.ip) + if err != nil { t.Errorf("failed to parse IP: %s", tc.ip) continue } var record map[string]interface{} - err := db.Lookup(ip, &record) + err = db.Lookup(addr).Decode(&record) if err != nil { t.Errorf("failed to lookup IP %s: %s", tc.ip, err.Error()) continue diff --git a/lib/cmd_metadata.go b/lib/cmd_metadata.go index 72408f7..cbbd48a 100644 --- a/lib/cmd_metadata.go +++ b/lib/cmd_metadata.go @@ -8,7 +8,7 @@ import ( "strings" "github.com/fatih/color" - "github.com/oschwald/maxminddb-golang" + "github.com/oschwald/maxminddb-golang/v2" "github.com/spf13/pflag" ) diff --git a/lib/cmd_read.go b/lib/cmd_read.go index 029b0bc..34e4590 100644 --- a/lib/cmd_read.go +++ b/lib/cmd_read.go @@ -4,11 +4,12 @@ import ( "encoding/csv" "encoding/json" "fmt" + "net/netip" "os" "github.com/fatih/color" "github.com/ipinfo/cli/lib/iputil" - "github.com/oschwald/maxminddb-golang" + "github.com/oschwald/maxminddb-golang/v2" "github.com/spf13/pflag" ) @@ -102,7 +103,17 @@ func CmdRead(f CmdReadFlags, args []string, printHelp func()) error { } for _, ip := range ips { record := make(map[string]interface{}) - if err := db.Lookup(ip, &record); err != nil || len(record) == 0 { + addr, ok := netip.AddrFromSlice(ip) + if !ok { + if !requiresHdr { + fmt.Fprintf(os.Stderr, + "err: invalid IP address %s\n", + ip.String(), + ) + } + continue + } + if err := db.Lookup(addr).Decode(&record); err != nil || len(record) == 0 { if !requiresHdr { fmt.Fprintf(os.Stderr, "err: couldn't get data for %s\n", diff --git a/lib/cmd_verify.go b/lib/cmd_verify.go index 0399b5e..d103644 100644 --- a/lib/cmd_verify.go +++ b/lib/cmd_verify.go @@ -4,7 +4,7 @@ import ( "errors" "fmt" - "github.com/oschwald/maxminddb-golang" + "github.com/oschwald/maxminddb-golang/v2" "github.com/spf13/pflag" ) From df7ed1ea096208fdaa9975401fab8324c628cfb1 Mon Sep 17 00:00:00 2001 From: Max Mouchet Date: Thu, 22 Jan 2026 21:37:07 +0100 Subject: [PATCH 3/5] update --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 6072399..6afac57 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/fatih/color v1.16.0 github.com/ipinfo/cli v0.0.0-20240814004006-a9ca4b1d939d github.com/maxmind/mmdbwriter v1.0.1-0.20231024181307-469cd9b959b4 - github.com/oschwald/maxminddb-golang/v2 v2.0.0 + github.com/oschwald/maxminddb-golang/v2 v2.1.1 github.com/spf13/pflag v1.0.5 ) @@ -20,5 +20,5 @@ require ( github.com/posener/script v1.2.0 // indirect go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 // indirect - golang.org/x/sys v0.37.0 // indirect + golang.org/x/sys v0.38.0 // indirect ) diff --git a/go.sum b/go.sum index 7c53025..084928c 100644 --- a/go.sum +++ b/go.sum @@ -20,8 +20,8 @@ github.com/maxmind/mmdbwriter v1.0.1-0.20231024181307-469cd9b959b4 h1:rF6qXloekm github.com/maxmind/mmdbwriter v1.0.1-0.20231024181307-469cd9b959b4/go.mod h1:6F/4tSDsJ8Y9UFVnehdZEIS220Uz62E7lbo8ZS0DehI= github.com/oschwald/maxminddb-golang v1.12.0 h1:9FnTOD0YOhP7DGxGsq4glzpGy5+w7pq50AS6wALUMYs= github.com/oschwald/maxminddb-golang v1.12.0/go.mod h1:q0Nob5lTCqyQ8WT6FYgS1L7PXKVVbgiymefNwIjPzgY= -github.com/oschwald/maxminddb-golang/v2 v2.0.0 h1:Gyljxck1kHbBxDgLM++NfDWBqvu1pWWfT8XbosSo0bo= -github.com/oschwald/maxminddb-golang/v2 v2.0.0/go.mod h1:gG4V88LsawPEqtbL1Veh1WRh+nVSYwXzJ1P5Fcn77g0= +github.com/oschwald/maxminddb-golang/v2 v2.1.1 h1:lA8FH0oOrM4u7mLvowq8IT6a3Q/qEnqRzLQn9eH5ojc= +github.com/oschwald/maxminddb-golang/v2 v2.1.1/go.mod h1:PLdx6PR+siSIoXqqy7C7r3SB3KZnhxWr1Dp6g0Hacl8= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/script v1.2.0 h1:DrZz0qFT8lCLkYNi1PleLDANFnKxJ2VmlNPJbAkVLsE= @@ -36,7 +36,7 @@ golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 h1:qCEDpW1G+vcj3Y7Fy52pEM1AW golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= -golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From dd57265fd14c79c1bc79e4bb0d8cb4e811ccc0d1 Mon Sep 17 00:00:00 2001 From: Max Mouchet Date: Fri, 23 Jan 2026 16:14:47 +0100 Subject: [PATCH 4/5] update --- lib/cmd_export.go | 132 +++++-------------------------------------- lib/exporter.go | 26 +++++++++ lib/exporter_csv.go | 70 +++++++++++++++++++++++ lib/exporter_json.go | 61 ++++++++++++++++++++ lib/exporter_tsv.go | 69 ++++++++++++++++++++++ 5 files changed, 239 insertions(+), 119 deletions(-) create mode 100644 lib/exporter.go create mode 100644 lib/exporter_csv.go create mode 100644 lib/exporter_json.go create mode 100644 lib/exporter_tsv.go diff --git a/lib/cmd_export.go b/lib/cmd_export.go index 550fd5e..adf6df8 100644 --- a/lib/cmd_export.go +++ b/lib/cmd_export.go @@ -1,9 +1,6 @@ package lib import ( - "bufio" - "encoding/csv" - "encoding/json" "errors" "fmt" "os" @@ -79,7 +76,7 @@ func CmdExport(f CmdExportFlags, args []string, printHelp func()) error { defer outFile.Close() } - // validate format. + // infer format from extension if not specified. if f.Format == "" { if strings.HasSuffix(f.Out, ".csv") { f.Format = "csv" @@ -91,9 +88,6 @@ func CmdExport(f CmdExportFlags, args []string, printHelp func()) error { f.Format = "csv" } } - if f.Format != "csv" && f.Format != "tsv" && f.Format != "json" { - return errors.New("format must be \"csv\" or \"tsv\" or \"json\"") - } // open tree. db, err := maxminddb.Open(args[0]) @@ -102,117 +96,17 @@ func CmdExport(f CmdExportFlags, args []string, printHelp func()) error { } defer db.Close() - if f.Format == "tsv" || f.Format == "csv" { - // export. - hdrWritten := false - var wr writer - if f.Format == "csv" { - csvwr := csv.NewWriter(outFile) - wr = csvwr - } else { - tsvwr := NewTsvWriter(outFile) - wr = tsvwr - } - - // Cache for decoded and serialized records, keyed by data offset. - // Many networks can point to the same data record in MMDB files. - cache := make(map[uintptr]map[string]string) - var hdrKeys []string - - for result := range db.Networks() { - if err := result.Err(); err != nil { - return fmt.Errorf("failed networks traversal: %w", err) - } - - offset := result.Offset() - prefix := result.Prefix() - - var recordStr map[string]string - if cached, ok := cache[offset]; ok { - recordStr = cached - } else { - // Cache miss: decode and serialize. - record := make(map[string]any) - if err := result.Decode(&record); err != nil { - return fmt.Errorf("failed to decode record: %w", err) - } - recordStr = mapInterfaceToStr(record) - cache[offset] = recordStr - } - - if !hdrWritten { - hdrWritten = true - hdrKeys = sortedMapKeys(recordStr) - if !f.NoHdr { - hdr := append([]string{"range"}, hdrKeys...) - if err := wr.Write(hdr); err != nil { - return fmt.Errorf( - "failed to write header %v: %w", - hdr, err, - ) - } - } - } - - // Build values in header key order, using empty string for missing keys. - vals := make([]string, len(hdrKeys)) - for i, k := range hdrKeys { - vals[i] = recordStr[k] // Returns "" if key doesn't exist - } - - line := append([]string{prefix.String()}, vals...) - if err := wr.Write(line); err != nil { - return fmt.Errorf("failed to write line %v: %w", line, err) - } - } - wr.Flush() - if err := wr.Error(); err != nil { - return fmt.Errorf("writer had failure: %w", err) - } - } else if f.Format == "json" { - // Cache for JSON-encoded records (without "range" field), keyed by data offset. - cache := make(map[uintptr][]byte) - bw := bufio.NewWriter(outFile) - - for result := range db.Networks() { - if err := result.Err(); err != nil { - return fmt.Errorf("failed networks traversal: %w", err) - } - - offset := result.Offset() - prefix := result.Prefix() - - var jsonSuffix []byte - if cached, ok := cache[offset]; ok { - jsonSuffix = cached - } else { - // Cache miss: decode and encode to JSON. - record := make(map[string]any) - if err := result.Decode(&record); err != nil { - return fmt.Errorf("failed to decode record: %w", err) - } - - encoded, err := json.Marshal(record) - if err != nil { - return fmt.Errorf("failed to encode record: %w", err) - } - // Cache everything after the opening '{'. - jsonSuffix = encoded[1:] - cache[offset] = jsonSuffix - } - - // Write: {"range":"",...}\n - // jsonSuffix is either "}" (empty record) or `"key":val,...}` - bw.WriteString(`{"range":"`) - bw.WriteString(prefix.String()) - bw.WriteString(`"`) - if len(jsonSuffix) > 1 { // More than just "}" - bw.WriteByte(',') - } - bw.Write(jsonSuffix) - bw.WriteByte('\n') - } - bw.Flush() + var exp exporter + switch f.Format { + case "csv": + exp = newCSVExporter(outFile, f.NoHdr) + case "tsv": + exp = newTSVExporter(outFile, f.NoHdr) + case "json": + exp = newJSONExporter(outFile) + default: + return errors.New("format must be \"csv\" or \"tsv\" or \"json\"") } - return nil + + return exportNetworks(db, exp) } diff --git a/lib/exporter.go b/lib/exporter.go new file mode 100644 index 0000000..7522c0b --- /dev/null +++ b/lib/exporter.go @@ -0,0 +1,26 @@ +package lib + +import ( + "fmt" + + "github.com/oschwald/maxminddb-golang/v2" +) + +// exporter defines the interface for exporting MMDB records. +type exporter interface { + WriteRecord(result maxminddb.Result) error + Flush() error +} + +// exportNetworks iterates over all networks in the database and writes them using the exporter. +func exportNetworks(db *maxminddb.Reader, exp exporter) error { + for result := range db.Networks() { + if err := result.Err(); err != nil { + return fmt.Errorf("failed networks traversal: %w", err) + } + if err := exp.WriteRecord(result); err != nil { + return err + } + } + return exp.Flush() +} diff --git a/lib/exporter_csv.go b/lib/exporter_csv.go new file mode 100644 index 0000000..f8bc2aa --- /dev/null +++ b/lib/exporter_csv.go @@ -0,0 +1,70 @@ +package lib + +import ( + "encoding/csv" + "fmt" + "io" + + "github.com/oschwald/maxminddb-golang/v2" +) + +// csvExporter exports records in CSV format. +type csvExporter struct { + wr *csv.Writer + cache map[uintptr]map[string]string + hdrKeys []string + noHdr bool +} + +func newCSVExporter(w io.Writer, noHdr bool) *csvExporter { + return &csvExporter{ + wr: csv.NewWriter(w), + cache: make(map[uintptr]map[string]string), + noHdr: noHdr, + } +} + +func (e *csvExporter) WriteRecord(result maxminddb.Result) error { + offset := result.Offset() + prefix := result.Prefix() + + var recordStr map[string]string + if cached, ok := e.cache[offset]; ok { + recordStr = cached + } else { + record := make(map[string]any) + if err := result.Decode(&record); err != nil { + return fmt.Errorf("failed to decode record: %w", err) + } + recordStr = mapInterfaceToStr(record) + e.cache[offset] = recordStr + } + + // Write header on first record. + if e.hdrKeys == nil { + e.hdrKeys = sortedMapKeys(recordStr) + if !e.noHdr { + hdr := append([]string{"range"}, e.hdrKeys...) + if err := e.wr.Write(hdr); err != nil { + return fmt.Errorf("failed to write header %v: %w", hdr, err) + } + } + } + + // Build values in header key order. + vals := make([]string, len(e.hdrKeys)) + for i, k := range e.hdrKeys { + vals[i] = recordStr[k] + } + + line := append([]string{prefix.String()}, vals...) + if err := e.wr.Write(line); err != nil { + return fmt.Errorf("failed to write line %v: %w", line, err) + } + return nil +} + +func (e *csvExporter) Flush() error { + e.wr.Flush() + return e.wr.Error() +} diff --git a/lib/exporter_json.go b/lib/exporter_json.go new file mode 100644 index 0000000..143e922 --- /dev/null +++ b/lib/exporter_json.go @@ -0,0 +1,61 @@ +package lib + +import ( + "bufio" + "encoding/json" + "fmt" + "io" + + "github.com/oschwald/maxminddb-golang/v2" +) + +// jsonExporter exports records in JSON Lines format. +type jsonExporter struct { + bw *bufio.Writer + cache map[uintptr][]byte +} + +func newJSONExporter(w io.Writer) *jsonExporter { + return &jsonExporter{ + bw: bufio.NewWriter(w), + cache: make(map[uintptr][]byte), + } +} + +func (e *jsonExporter) WriteRecord(result maxminddb.Result) error { + offset := result.Offset() + prefix := result.Prefix() + + var jsonSuffix []byte + if cached, ok := e.cache[offset]; ok { + jsonSuffix = cached + } else { + record := make(map[string]any) + if err := result.Decode(&record); err != nil { + return fmt.Errorf("failed to decode record: %w", err) + } + + encoded, err := json.Marshal(record) + if err != nil { + return fmt.Errorf("failed to encode record: %w", err) + } + // Cache everything after the opening '{'. + jsonSuffix = encoded[1:] + e.cache[offset] = jsonSuffix + } + + // Write: {"range":"",...}\n + e.bw.WriteString(`{"range":"`) + e.bw.WriteString(prefix.String()) + e.bw.WriteString(`"`) + if len(jsonSuffix) > 1 { // More than just "}" + e.bw.WriteByte(',') + } + e.bw.Write(jsonSuffix) + e.bw.WriteByte('\n') + return nil +} + +func (e *jsonExporter) Flush() error { + return e.bw.Flush() +} diff --git a/lib/exporter_tsv.go b/lib/exporter_tsv.go new file mode 100644 index 0000000..a4e12a0 --- /dev/null +++ b/lib/exporter_tsv.go @@ -0,0 +1,69 @@ +package lib + +import ( + "fmt" + "io" + + "github.com/oschwald/maxminddb-golang/v2" +) + +// tsvExporter exports records in TSV format. +type tsvExporter struct { + wr *TsvWriter + cache map[uintptr]map[string]string + hdrKeys []string + noHdr bool +} + +func newTSVExporter(w io.Writer, noHdr bool) *tsvExporter { + return &tsvExporter{ + wr: NewTsvWriter(w), + cache: make(map[uintptr]map[string]string), + noHdr: noHdr, + } +} + +func (e *tsvExporter) WriteRecord(result maxminddb.Result) error { + offset := result.Offset() + prefix := result.Prefix() + + var recordStr map[string]string + if cached, ok := e.cache[offset]; ok { + recordStr = cached + } else { + record := make(map[string]any) + if err := result.Decode(&record); err != nil { + return fmt.Errorf("failed to decode record: %w", err) + } + recordStr = mapInterfaceToStr(record) + e.cache[offset] = recordStr + } + + // Write header on first record. + if e.hdrKeys == nil { + e.hdrKeys = sortedMapKeys(recordStr) + if !e.noHdr { + hdr := append([]string{"range"}, e.hdrKeys...) + if err := e.wr.Write(hdr); err != nil { + return fmt.Errorf("failed to write header %v: %w", hdr, err) + } + } + } + + // Build values in header key order. + vals := make([]string, len(e.hdrKeys)) + for i, k := range e.hdrKeys { + vals[i] = recordStr[k] + } + + line := append([]string{prefix.String()}, vals...) + if err := e.wr.Write(line); err != nil { + return fmt.Errorf("failed to write line %v: %w", line, err) + } + return nil +} + +func (e *tsvExporter) Flush() error { + e.wr.Flush() + return e.wr.Error() +} From 5418fc5dff7e50b3f402b12815e186db52bee392 Mon Sep 17 00:00:00 2001 From: Max Mouchet Date: Fri, 23 Jan 2026 16:26:34 +0100 Subject: [PATCH 5/5] update --- lib/exporter_json.go | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/lib/exporter_json.go b/lib/exporter_json.go index 143e922..a709a3a 100644 --- a/lib/exporter_json.go +++ b/lib/exporter_json.go @@ -2,6 +2,7 @@ package lib import ( "bufio" + "bytes" "encoding/json" "fmt" "io" @@ -9,6 +10,8 @@ import ( "github.com/oschwald/maxminddb-golang/v2" ) +const rangePlaceholder = "__RANGE__" + // jsonExporter exports records in JSON Lines format. type jsonExporter struct { bw *bufio.Writer @@ -26,32 +29,24 @@ func (e *jsonExporter) WriteRecord(result maxminddb.Result) error { offset := result.Offset() prefix := result.Prefix() - var jsonSuffix []byte - if cached, ok := e.cache[offset]; ok { - jsonSuffix = cached - } else { + cached, ok := e.cache[offset] + if !ok { record := make(map[string]any) if err := result.Decode(&record); err != nil { return fmt.Errorf("failed to decode record: %w", err) } + record["range"] = rangePlaceholder encoded, err := json.Marshal(record) if err != nil { return fmt.Errorf("failed to encode record: %w", err) } - // Cache everything after the opening '{'. - jsonSuffix = encoded[1:] - e.cache[offset] = jsonSuffix + cached = encoded + e.cache[offset] = cached } - // Write: {"range":"",...}\n - e.bw.WriteString(`{"range":"`) - e.bw.WriteString(prefix.String()) - e.bw.WriteString(`"`) - if len(jsonSuffix) > 1 { // More than just "}" - e.bw.WriteByte(',') - } - e.bw.Write(jsonSuffix) + line := bytes.Replace(cached, []byte(rangePlaceholder), []byte(prefix.String()), 1) + e.bw.Write(line) e.bw.WriteByte('\n') return nil }