From 81eafb8a4b3023d1ab42d4d5fffe3a8f3dc006c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20BERSAC?= Date: Wed, 22 Oct 2025 15:41:03 +0200 Subject: [PATCH 1/7] cmd: Show HWM instead of VMPeak HWM is more readable. --- docs/changelog.md | 5 +++++ internal/cmd/flags.go | 4 ++-- internal/perf/mem.go | 8 ++++---- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 48c38cf9..4af83534 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -11,6 +11,11 @@ Here is a highlight of changes in each versions. If you need further details, follow [merged Pull request pages](https://github.com/dalibo/ldap2pg/pulls?utf8=%E2%9C%93&q=is%3Apr%20is%3Amerged). +# UNRELEASED + +- Show high watermark instead of VMPeak. + + # ldap2pg 6.5.1 - Fix inspection of grants on functions. Thanks @dani. diff --git a/internal/cmd/flags.go b/internal/cmd/flags.go index 7f931821..5c687b67 100644 --- a/internal/cmd/flags.go +++ b/internal/cmd/flags.go @@ -108,11 +108,11 @@ func (controller Controller) Finalize(errs *errorlist.List, start time.Time, rol "grants", grants, ) } - vmPeak := perf.ReadVMPeak() + mem := perf.ReadHVM() elapsed := time.Since(start) logAttrs = append(logAttrs, "elapsed", elapsed, - "mempeak", perf.FormatBytes(vmPeak), + "mem", perf.FormatBytes(mem), "ldap", ldap.Watch.Total, "inspect", inspect.Watch.Total, "sync", postgres.Watch.Total, diff --git a/internal/perf/mem.go b/internal/perf/mem.go index f866dfec..92ce54f0 100644 --- a/internal/perf/mem.go +++ b/internal/perf/mem.go @@ -9,7 +9,7 @@ import ( "strings" ) -func ReadVMPeak() int { +func ReadHVM() int { fo, err := os.Open("/proc/self/status") if err != nil { slog.Debug("Failed to read /proc/self/status.", "err", err) @@ -20,17 +20,17 @@ func ReadVMPeak() int { scanner := bufio.NewScanner(fo) for scanner.Scan() { line := scanner.Text() - if !strings.HasPrefix(line, "VmPeak:") { + if !strings.HasPrefix(line, "VmHWM:") { continue } fields := strings.Fields(line) value, err := strconv.Atoi(fields[1]) if err != nil { - slog.Debug("Failed to parse VmPeak.", "err", err) + slog.Debug("Failed to parse VmHWM.", "err", err) return 0 } - return value + return value * 1024 } if err := scanner.Err(); err != nil { From 52adb13c24b6fb4d8fe1012849f2fa7901a23826 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20BERSAC?= Date: Wed, 22 Oct 2025 15:56:13 +0200 Subject: [PATCH 2/7] cmd: More profiling and tracing --- .gitignore | 2 ++ docs/changelog.md | 1 + go.mod | 3 +++ go.sum | 15 +++++++++++++++ internal/cmd/ldap2pg.go | 31 ++----------------------------- internal/cmd/profile.go | 39 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 62 insertions(+), 29 deletions(-) create mode 100644 internal/cmd/profile.go diff --git a/.gitignore b/.gitignore index f1bdf175..b1ddd107 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ dist/ docker-compose.override.yml # test/conftest.py creates .env files .env +trace.out +*.pprof diff --git a/docs/changelog.md b/docs/changelog.md index 4af83534..d9e759db 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -14,6 +14,7 @@ pages](https://github.com/dalibo/ldap2pg/pulls?utf8=%E2%9C%93&q=is%3Apr%20is%3Am # UNRELEASED - Show high watermark instead of VMPeak. +- Profile memory and trace execution. # ldap2pg 6.5.1 diff --git a/go.mod b/go.mod index 7c92984f..24f7c881 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( github.com/lmittmann/tint v1.1.2 github.com/mattn/go-isatty v0.0.20 github.com/mitchellh/mapstructure v1.5.0 + github.com/pkg/profile v1.7.0 github.com/spf13/pflag v1.0.10 github.com/stretchr/testify v1.10.0 golang.org/x/exp v0.0.0-20250911091902-df9299821621 @@ -30,8 +31,10 @@ require ( require ( github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/felixge/fgprof v0.9.3 // indirect github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect + github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect github.com/google/uuid v1.6.0 // indirect github.com/gosimple/unidecode v1.0.1 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect diff --git a/go.sum b/go.sum index ee07c2dd..8b9b8148 100644 --- a/go.sum +++ b/go.sum @@ -4,18 +4,26 @@ github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7V github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4= github.com/avast/retry-go/v4 v4.6.1 h1:VkOLRubHdisGrHnTu89g08aQEWEgRU7LVEop3GbIcMk= github.com/avast/retry-go/v4 v4.6.1/go.mod h1:V6oF8njAwxJ5gRo1Q7Cxab24xs5NCWZBeaHHBklR8mA= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckarep/golang-set/v2 v2.8.0 h1:swm0rlPCmdWn9mESxKOjWk8hXSqoxOp+ZlfuyaAdFlQ= github.com/deckarep/golang-set/v2 v2.8.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= +github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g= +github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw= github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 h1:BP4M0CvQ4S3TGls2FvczZtj5Re/2ZzkV9VwqPHH/3Bo= github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= github.com/go-ldap/ldap/v3 v3.4.11 h1:4k0Yxweg+a3OyBLjdYn5OKglv18JNvfDykSoI8bW0gU= github.com/go-ldap/ldap/v3 v3.4.11/go.mod h1:bY7t0FLK8OAVpp/vV6sSlpz3EQDGcQwc8pF0ujLgKvM= github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/google/pprof v0.0.0-20211214055906-6f57359322fd h1:1FjCyPC+syAzJ5/2S8fqdZK1R22vvA0J7JZKcuOIQ7Y= +github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gosimple/slug v1.15.0 h1:wRZHsRrRcs6b0XnxMUBM6WK1U1Vg5B0R7VkIf1Xzobo= @@ -24,6 +32,7 @@ github.com/gosimple/unidecode v1.0.1 h1:hZzFTMMqSswvf0LBJZCZgThIZrpDHFXux9KeGmn6 github.com/gosimple/unidecode v1.0.1/go.mod h1:CP0Cr1Y1kogOtx0bJblKzsVWrqYaqfNOnHzpgWw4Awc= github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= @@ -73,6 +82,8 @@ github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/profile v1.7.0 h1:hnbDkaNWPCLMO9wGLdBFTIZvzDrDfBM2072E1S9gJkA= +github.com/pkg/profile v1.7.0/go.mod h1:8Uer0jas47ZQMJ7VD+OHknK4YDY07LPUC6dEvqDjvNo= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -81,8 +92,11 @@ github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/f github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI= @@ -93,6 +107,7 @@ golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= diff --git a/internal/cmd/ldap2pg.go b/internal/cmd/ldap2pg.go index 2beeb647..761a2d4f 100644 --- a/internal/cmd/ldap2pg.go +++ b/internal/cmd/ldap2pg.go @@ -8,12 +8,10 @@ import ( "os" "runtime" "runtime/debug" - "runtime/pprof" "strings" "time" "golang.org/x/exp/maps" - "golang.org/x/exp/slices" "github.com/dalibo/ldap2pg/v6/internal" "github.com/dalibo/ldap2pg/v6/internal/config" @@ -70,12 +68,9 @@ func ldap2pg() (err error) { start := time.Now() - stop, err := startProfiling() - if err != nil { - return - } + stop := startProfiling() if stop != nil { - defer stop() + defer stop.Stop() } defer postgres.CloseConn(ctx) @@ -315,25 +310,3 @@ func logPanic() { slog.Error("Please file an issue at https://github.com/dalibo/ldap2pg/issue/new with verbose log.") os.Exit(1) } - -func startProfiling() (stop func(), err error) { - if !slices.Contains(os.Environ(), "CPUPROFILE=1") { - return - } - slog.Debug("Starting CPU profiling.") - f, err := os.Create("default.pgo") - if err != nil { - return - } - err = pprof.StartCPUProfile(f) - if err != nil { - _ = f.Close() - return - } - stop = func() { - slog.Debug("Stopping profiling.") - pprof.StopCPUProfile() - _ = f.Close() - } - return -} diff --git a/internal/cmd/profile.go b/internal/cmd/profile.go new file mode 100644 index 00000000..ed84f201 --- /dev/null +++ b/internal/cmd/profile.go @@ -0,0 +1,39 @@ +package cmd + +import ( + "log/slog" + "os" + + "github.com/pkg/profile" +) + +func startProfiling() stopper { + pprof := os.Getenv("LDAP2PG_PROFILE") + if pprof == "" { + return nil + } + options := []func(*profile.Profile){profile.ProfilePath(".")} + switch pprof { + case "block": + options = append(options, profile.BlockProfile) + case "clock": + options = append(options, profile.ClockProfile) + case "goroutine": + options = append(options, profile.GoroutineProfile) + case "mem": + options = append(options, profile.MemProfile, profile.MemProfileRate(1024)) + case "mutex": + options = append(options, profile.MutexProfile) + case "trace": + options = append(options, profile.TraceProfile) + case "cpu": + options = append(options, profile.CPUProfile) + default: + slog.Warn("Unknown profile type.", "type", pprof) + } + return profile.Start(options...) +} + +type stopper interface { + Stop() +} From 52d6665b9254e1d788bd3588726b3e1c2d3690ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20BERSAC?= Date: Wed, 22 Oct 2025 16:10:06 +0200 Subject: [PATCH 3/7] Fallback description to rules item number --- internal/wanted/{step.go => item.go} | 83 ++++++++++--------- .../wanted/{step_test.go => item_test.go} | 0 internal/wanted/map.go | 11 +-- 3 files changed, 52 insertions(+), 42 deletions(-) rename internal/wanted/{step.go => item.go} (70%) rename internal/wanted/{step_test.go => item_test.go} (100%) diff --git a/internal/wanted/step.go b/internal/wanted/item.go similarity index 70% rename from internal/wanted/step.go rename to internal/wanted/item.go index 10886a8e..8eabde54 100644 --- a/internal/wanted/step.go +++ b/internal/wanted/item.go @@ -1,6 +1,7 @@ package wanted import ( + "fmt" "log/slog" "strings" @@ -13,26 +14,34 @@ import ( "golang.org/x/exp/slices" ) -type Step struct { +type RulesItem struct { + pos int Description string LdapSearch ldap.Search RoleRules []RoleRule `mapstructure:"roles"` GrantRules []privileges.GrantRule `mapstructure:"grants"` } -func (s Step) HasLDAPSearch() bool { - return len(s.LdapSearch.Attributes) > 0 +func (item RulesItem) String() string { + if item.Description == "" { + return fmt.Sprintf("%d", item.pos) + } + return item.Description +} + +func (item RulesItem) HasLDAPSearch() bool { + return len(item.LdapSearch.Attributes) > 0 } -func (s Step) HasSubsearch() bool { - return len(s.LdapSearch.Subsearches) > 0 +func (item RulesItem) HasSubsearch() bool { + return len(item.LdapSearch.Subsearches) > 0 } -func (s *Step) InferAttributes() { +func (item *RulesItem) InferAttributes() { attributes := mapset.NewSet[string]() subsearchAttributes := make(map[string]mapset.Set[string]) - for field := range s.IterFields() { + for field := range item.IterFields() { attribute, field, found := strings.Cut(field.FieldName, ".") // dn is the primary key of the entry, not a real attribute. if attribute == "dn" { @@ -58,19 +67,19 @@ func (s *Step) InferAttributes() { return } - s.LdapSearch.Attributes = attributes.ToSlice() + item.LdapSearch.Attributes = attributes.ToSlice() slog.Debug("Collected LDAP search attributes.", - "item", s.Description, "base", s.LdapSearch.Base, "attributes", s.LdapSearch.Attributes) + "item", item, "base", item.LdapSearch.Base, "attributes", item.LdapSearch.Attributes) if len(subsearchAttributes) == 0 { return } - if s.LdapSearch.Subsearches == nil { - s.LdapSearch.Subsearches = make(map[string]ldap.Subsearch) + if item.LdapSearch.Subsearches == nil { + item.LdapSearch.Subsearches = make(map[string]ldap.Subsearch) } for attribute, subAttributes := range subsearchAttributes { - subsearch, ok := s.LdapSearch.Subsearches[attribute] + subsearch, ok := item.LdapSearch.Subsearches[attribute] if !ok { subsearch = ldap.Subsearch{ Filter: "(objectClass=*)", @@ -79,15 +88,15 @@ func (s *Step) InferAttributes() { } subsearch.Attributes = subAttributes.ToSlice() slog.Debug("Collected LDAP sub-search attributes.", - "item", s.Description, "base", s.LdapSearch.Base, + "item", item, "base", item.LdapSearch.Base, "fkey", attribute, "attributes", subsearch.Attributes) - s.LdapSearch.Subsearches[attribute] = subsearch + item.LdapSearch.Subsearches[attribute] = subsearch } } -func (s *Step) ReplaceAttributeAsSubentryField() { - subsearchAttr := s.LdapSearch.SubsearchAttribute() - for field := range s.IterFields() { +func (item *RulesItem) ReplaceAttributeAsSubentryField() { + subsearchAttr := item.LdapSearch.SubsearchAttribute() + for field := range item.IterFields() { attribute, _, found := strings.Cut(field.FieldName, ".") if attribute != subsearchAttr { continue @@ -104,18 +113,18 @@ func (s *Step) ReplaceAttributeAsSubentryField() { } // Yields all {attr} from all formats in item. -func (s Step) IterFields() <-chan *pyfmt.Field { +func (item RulesItem) IterFields() <-chan *pyfmt.Field { ch := make(chan *pyfmt.Field) go func() { defer close(ch) - for _, rule := range s.RoleRules { + for _, rule := range item.RoleRules { for _, f := range rule.Formats() { for _, field := range f.Fields { ch <- field } } } - for _, rule := range s.GrantRules { + for _, rule := range item.GrantRules { for _, f := range rule.Formats() { for _, field := range f.Fields { ch <- field @@ -126,9 +135,9 @@ func (s Step) IterFields() <-chan *pyfmt.Field { return ch } -func (s Step) SplitStaticItems() (items []Step) { +func (item RulesItem) SplitStaticItems() (items []RulesItem) { var staticRoles, dynamicRoles []RoleRule - for _, rule := range s.RoleRules { + for _, rule := range item.RoleRules { if rule.IsStatic() { staticRoles = append(staticRoles, rule) } else { @@ -136,7 +145,7 @@ func (s Step) SplitStaticItems() (items []Step) { } } var staticGrants, dynamicGrants []privileges.GrantRule - for _, rule := range s.GrantRules { + for _, rule := range item.GrantRules { if rule.IsStatic() { staticGrants = append(staticGrants, rule) } else { @@ -146,18 +155,18 @@ func (s Step) SplitStaticItems() (items []Step) { if (len(staticRoles) == 0 && len(staticGrants) == 0) || (len(dynamicRoles) == 0 && len(dynamicGrants) == 0) { - items = append(items, s) + items = append(items, item) return } - items = append(items, Step{ - Description: s.Description, - LdapSearch: s.LdapSearch, + items = append(items, RulesItem{ + Description: item.Description, + LdapSearch: item.LdapSearch, RoleRules: dynamicRoles, GrantRules: dynamicGrants, }) - items = append(items, Step{ + items = append(items, RulesItem{ // Avoid duplicating log message, use a silent item. Description: "", RoleRules: staticRoles, @@ -174,23 +183,23 @@ type SearchResult struct { // search directory, returning each entry or error. Sub-searches are done // concurrently and returned for each sub-key. -func (s Step) search(ldapc ldap.Client) <-chan SearchResult { +func (item RulesItem) search(ldapc ldap.Client) <-chan SearchResult { ch := make(chan SearchResult) go func() { defer close(ch) - if !s.HasLDAPSearch() { + if !item.HasLDAPSearch() { // Use a dumb empty result. ch <- SearchResult{} return } - search := s.LdapSearch + search := item.LdapSearch res, err := ldapc.Search(search.Base, search.Scope, search.Filter, search.Attributes) if err != nil { ch <- SearchResult{err: err} return } - subsearchAttr := s.LdapSearch.SubsearchAttribute() + subsearchAttr := item.LdapSearch.SubsearchAttribute() for _, entry := range res.Entries { slog.Debug("Got LDAP entry.", "dn", entry.DN) result := ldap.Result{ @@ -203,7 +212,7 @@ func (s Step) search(ldapc ldap.Client) <-chan SearchResult { } bases := entry.GetEqualFoldAttributeValues(subsearchAttr) for _, base := range bases { - s := s.LdapSearch.Subsearches[subsearchAttr] + s := item.LdapSearch.Subsearches[subsearchAttr] res, err = ldapc.Search(base, s.Scope, s.Filter, s.Attributes) if err != nil { ch <- SearchResult{err: err} @@ -220,11 +229,11 @@ func (s Step) search(ldapc ldap.Client) <-chan SearchResult { return ch } -func (s Step) generateRoles(results *ldap.Result) <-chan role.Role { +func (item RulesItem) generateRoles(results *ldap.Result) <-chan role.Role { ch := make(chan role.Role) go func() { defer close(ch) - for _, rule := range s.RoleRules { + for _, rule := range item.RoleRules { for role := range rule.Generate(results) { ch <- role } @@ -233,11 +242,11 @@ func (s Step) generateRoles(results *ldap.Result) <-chan role.Role { return ch } -func (s Step) generateGrants(results *ldap.Result) <-chan privileges.Grant { +func (item RulesItem) generateGrants(results *ldap.Result) <-chan privileges.Grant { ch := make(chan privileges.Grant) go func() { defer close(ch) - for _, rule := range s.GrantRules { + for _, rule := range item.GrantRules { for grant := range rule.Generate(results) { ch <- grant } diff --git a/internal/wanted/step_test.go b/internal/wanted/item_test.go similarity index 100% rename from internal/wanted/step_test.go rename to internal/wanted/item_test.go diff --git a/internal/wanted/map.go b/internal/wanted/map.go index 52e27652..6b6e08d1 100644 --- a/internal/wanted/map.go +++ b/internal/wanted/map.go @@ -12,7 +12,7 @@ import ( ) // Rules holds a set of rules to generate wanted state. -type Rules []Step +type Rules []RulesItem func (m Rules) HasLDAPSearches() bool { for _, item := range m { @@ -23,10 +23,10 @@ func (m Rules) HasLDAPSearches() bool { return false } -func (m Rules) SplitStaticRules() (newMap Rules) { - newMap = make(Rules, 0) +func (m Rules) SplitStaticRules() (newRules Rules) { + newRules = make(Rules, 0) for _, item := range m { - newMap = append(newMap, item.SplitStaticItems()...) + newRules = append(newRules, item.SplitStaticItems()...) } return } @@ -58,10 +58,11 @@ func (m Rules) Run(blacklist lists.Blacklist) (roles role.Map, grants map[string roles = make(map[string]role.Role) grants = make(map[string][]privileges.Grant) for i, item := range m { + item.pos = i if item.Description != "" { slog.Info(item.Description) } else { - slog.Debug("Processing sync map item.", "item", i) + slog.Debug("Processing sync map step.", "item", item) } for res := range item.search(ldapc) { From 44bb143fef82d1524f684dc6df7c63a119de0b2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20BERSAC?= Date: Thu, 23 Oct 2025 08:56:12 +0200 Subject: [PATCH 4/7] docs: Update compose override sample --- docs/hacking.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/hacking.md b/docs/hacking.md index 71368028..6438f021 100644 --- a/docs/hacking.md +++ b/docs/hacking.md @@ -78,8 +78,6 @@ exposes containers ports to your host with the following override: ``` yaml # contents docker-compose.override.yml -version: '3' - services: samba: ports: From b2fb0bbad1d1cd617bcc8cdbac83313dd4f9075c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20BERSAC?= Date: Fri, 24 Oct 2025 11:24:55 +0200 Subject: [PATCH 5/7] sql: Reindent creators --- internal/inspect/sql/creators.sql | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/inspect/sql/creators.sql b/internal/inspect/sql/creators.sql index c93c5139..f212b29f 100644 --- a/internal/inspect/sql/creators.sql +++ b/internal/inspect/sql/creators.sql @@ -1,6 +1,6 @@ SELECT nspname, array_agg(rolname ORDER BY rolname) AS creators -FROM pg_catalog.pg_namespace AS nsp -CROSS JOIN pg_catalog.pg_roles AS creator -WHERE has_schema_privilege(creator.oid, nsp.oid, 'CREATE') - AND rolcanlogin -GROUP BY nspname; + FROM pg_catalog.pg_namespace AS nsp + CROSS JOIN pg_catalog.pg_roles AS creator + WHERE has_schema_privilege(creator.oid, nsp.oid, 'CREATE') + AND rolcanlogin + GROUP BY 1; From 0cd2d5ce41fe4797ed494bf1785dc5f0ac855b94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20BERSAC?= Date: Fri, 24 Oct 2025 11:51:17 +0200 Subject: [PATCH 6/7] pif --- toto | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 toto diff --git a/toto b/toto new file mode 100644 index 00000000..e69de29b From 97f758446ef7ee8212b49a319b0439fea9781c29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20BERSAC?= Date: Tue, 28 Oct 2025 11:49:06 +0100 Subject: [PATCH 7/7] =?UTF-8?q?d=C3=A9v:=20Review=20big=20case?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - fix user creation - increase user - increase namespaces - synchronize a single database by default - add more users in groups --- Makefile | 11 ++++++----- test/fixtures/big.sh | 2 +- test/fixtures/{genperfldif.sh => genbigldif.sh} | 17 ++++++++--------- test/genbigconfig.sh | 14 ++++++-------- 4 files changed, 21 insertions(+), 23 deletions(-) rename test/fixtures/{genperfldif.sh => genbigldif.sh} (73%) diff --git a/Makefile b/Makefile index 14306fa5..cf8452e7 100644 --- a/Makefile +++ b/Makefile @@ -4,13 +4,14 @@ YUM_LABS?=$(wildcard ../yum-labs) default: @echo ldap2pg $(VERSION) -big: reset-samba - while ! bash -c "echo -n > /dev/tcp/$${LDAPURI#*//}/636" ; do sleep 1; done +big: big-samba big-postgres + +big-samba: + while ! ldapwhoami -xw $$LDAPPASSWORD ; do sleep 1; done test/fixtures/genbigldif.sh | ldapmodify -xw $$LDAPPASSWORD - $(MAKE) reset-big -reset-big: reset-postgres - while ! bash -c "echo -n > /dev/tcp/$${PGHOST}/5432" ; do sleep 1; done +big-postgres: + while ! psql -U postgres -d postgres -l >/dev/null; do sleep 1; done test/fixtures/big.sh reset-%: diff --git a/test/fixtures/big.sh b/test/fixtures/big.sh index 7275e963..8a3a0e6b 100755 --- a/test/fixtures/big.sh +++ b/test/fixtures/big.sh @@ -16,7 +16,7 @@ CREATE DATABASE "big0" WITH OWNER "bigowner"; EOSQL queries=() -for i in {0..255} ; do +for i in {0..512} ; do printf -v i "%03d" "$i" queries+=("CREATE SCHEMA nsp$i AUTHORIZATION bigowner") for j in {0..3} ; do diff --git a/test/fixtures/genperfldif.sh b/test/fixtures/genbigldif.sh similarity index 73% rename from test/fixtures/genperfldif.sh rename to test/fixtures/genbigldif.sh index 359e93e0..82e32639 100755 --- a/test/fixtures/genperfldif.sh +++ b/test/fixtures/genbigldif.sh @@ -7,8 +7,8 @@ version: 2 charset: UTF-8 EOF -for i in {0..1023} ; do - printf -v u "u%04d" "$i" +for i in {0..16384} ; do + printf -v u "u%04x" "$i" cat <<-EOF dn: cn=$u,cn=users,dc=bridoulou,dc=fr @@ -23,7 +23,7 @@ for i in {0..1023} ; do EOF done -for i in {0..255} ; do +for i in {0..512} ; do printf -v base "big%03d_" "$i" for g in r w d ; do g="${base}$g" @@ -36,20 +36,19 @@ for i in {0..255} ; do cn: $g EOF - for u in {0..1023} ; do - break - if [ $((RANDOM % 128)) -gt 0 ] ; then + for u in {0..16384} ; do + if [ $((RANDOM % 1024)) -gt 0 ] ; then continue fi - printf -v u "u%04d" "$u" + printf -v u "u%04x" "$u" cat <<-EOF member: cn=$u,cn=users,dc=bridoulou,dc=fr EOF done # If no user has been added, add a random one. - if [ -n "${u#u*}" ] ; then - printf -v u "u%04d" "$(( RANDOM % 1024 ))" + if [ -z "${u#u*}" ] ; then + printf -v u "u%04x" "$(( RANDOM % 1024 ))" cat <<-EOF member: cn=$u,cn=users,dc=bridoulou,dc=fr EOF diff --git a/test/genbigconfig.sh b/test/genbigconfig.sh index 4f88f467..d5e1d498 100755 --- a/test/genbigconfig.sh +++ b/test/genbigconfig.sh @@ -6,7 +6,7 @@ cat <<-EOF version: 6 postgres: - databases_query: [big0, big1, big2, big3] + databases_query: [big0] managed_roles_query: | SELECT 'public' UNION @@ -49,7 +49,7 @@ rules: comment: All roles managed by ldap2pg EOF -for n in {0..255} ; do +for n in {0..512} ; do printf -v n "%03d" "$n" cat <<-EOF @@ -60,17 +60,15 @@ for n in {0..255} ; do - name: big${n}_w parents: - ldap_roles - - big${n}_r - name: big${n}_d parents: - ldap_roles - - big${n}_w grants: - privilege: read - role: big${n}_r + roles: [big${n}_r, big${n}_w, big${n}_d] schemas: nsp$n - privilege: write - role: big${n}_w + roles: [big${n}_w, big${n}_d] schemas: nsp$n - privilege: define role: big${n}_d @@ -79,13 +77,13 @@ for n in {0..255} ; do done cat <<-EOF - - description: "Define roles from directory." ldapsearch: base: cn=users,dc=bridoulou,dc=fr filter: (cn=big*) roles: - name: "{member.cn}" + - name: "{member.cn}" + options: LOGIN parents: - ldap_roles - "{cn}"