From 5831e125e536f0a0ed7e81a0e61ecf0fa6c9a6fd Mon Sep 17 00:00:00 2001 From: Nofre Date: Sat, 17 May 2025 18:29:11 +0200 Subject: [PATCH 1/6] chore(cmd): small adjustment in cmd --- cmd/cli/apply.go | 11 ++++++++++- cmd/cli/root.go | 4 ---- cmd/cli/version.go | 7 +++++-- go.mod | 9 ++++++++- go.sum | 16 ++++++++++++++++ ui/ui.go | 3 ++- 6 files changed, 41 insertions(+), 9 deletions(-) diff --git a/cmd/cli/apply.go b/cmd/cli/apply.go index 603e41f..b6112b9 100644 --- a/cmd/cli/apply.go +++ b/cmd/cli/apply.go @@ -8,7 +8,7 @@ import ( ) func init() { - applyCmd.Flags().StringP("file", "f", ".", "Path to manifest file") + applyCmd.Flags().StringP("file", "f", ".", "Path to manifests file") rootCmd.AddCommand(applyCmd) } @@ -46,3 +46,12 @@ var applyCmd = &cobra.Command{ ui.Println("Manifests applied successfully") }, } + +/*type Logger struct { + Writer io.Writer + Style lipgloss.Style +} + +func (l *Logger) Log(msg string) { + fmt.Fprintln(l.Writer, l.Style.Render(msg)) +}*/ diff --git a/cmd/cli/root.go b/cmd/cli/root.go index 0c9fce7..31852d8 100644 --- a/cmd/cli/root.go +++ b/cmd/cli/root.go @@ -12,7 +12,3 @@ var rootCmd = &cobra.Command{ func Execute() { cobra.CheckErr(rootCmd.Execute()) } - -func init() { - rootCmd.AddCommand(versionCmd) -} diff --git a/cmd/cli/version.go b/cmd/cli/version.go index 38acbc7..fd16177 100644 --- a/cmd/cli/version.go +++ b/cmd/cli/version.go @@ -2,7 +2,6 @@ package cli import ( "fmt" - "github.com/spf13/cobra" ) @@ -14,6 +13,10 @@ var versionCmd = &cobra.Command{ SilenceUsage: true, SilenceErrors: true, Run: func(cmd *cobra.Command, args []string) { - fmt.Println("Qube CLI Version: ", version) + fmt.Println("Qube CLI", version) }, } + +func init() { + rootCmd.AddCommand(versionCmd) +} diff --git a/go.mod b/go.mod index 17d77b7..cddb245 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,8 @@ require ( require ( github.com/RoaringBitmap/roaring/v2 v2.4.5 // indirect + github.com/VividCortex/ewma v1.2.0 // indirect + github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/bits-and-blooms/bitset v1.22.0 // indirect github.com/blevesearch/bleve_index_api v1.2.8 // indirect @@ -48,17 +50,21 @@ require ( github.com/google/flatbuffers v25.2.10+incompatible // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v0.0.0-20171115153421-f7279a603ede // indirect + github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 // indirect github.com/klauspost/compress v1.18.0 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect + github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/mschoch/smat v0.2.0 // indirect github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/termenv v0.16.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect + github.com/schollz/progressbar/v3 v3.18.0 // indirect github.com/spf13/pflag v1.0.6 // indirect + github.com/vbauerster/mpb/v8 v8.10.1 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect go.etcd.io/bbolt v1.4.0 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect @@ -67,7 +73,8 @@ require ( go.opentelemetry.io/otel/trace v1.35.0 // indirect golang.org/x/net v0.38.0 // indirect golang.org/x/sync v0.13.0 // indirect - golang.org/x/sys v0.32.0 // indirect + golang.org/x/sys v0.33.0 // indirect + golang.org/x/term v0.32.0 // indirect golang.org/x/text v0.23.0 // indirect google.golang.org/protobuf v1.36.6 // indirect ) diff --git a/go.sum b/go.sum index e9d6318..2bac064 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,9 @@ github.com/RoaringBitmap/roaring/v2 v2.4.5 h1:uGrrMreGjvAtTBobc0g5IrW1D5ldxDQYe2JW2gggRdg= github.com/RoaringBitmap/roaring/v2 v2.4.5/go.mod h1:FiJcsfkGje/nZBZgCu0ZxCPOKD/hVXDS2dXi7/eUFE0= +github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= +github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/adrg/xdg v0.5.3 h1:xRnxJXne7+oWDatRhR1JLnvuccuIeCoBu2rtuLqQB78= github.com/adrg/xdg v0.5.3/go.mod h1:nlTsY+NNiCBGCK2tpm09vRqfVzrc2fLmXGpBLF0zlTQ= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= @@ -93,6 +97,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/json-iterator/go v0.0.0-20171115153421-f7279a603ede h1:YrgBGwxMRK0Vq0WSCWFaZUnTsrA/PZE/xs1QZh+/edg= github.com/json-iterator/go v0.0.0-20171115153421-f7279a603ede/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 h1:qGQQKEcAR99REcMpsXCp3lJ03zYT1PkRd3kQGPn9GVg= +github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -107,6 +113,8 @@ github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2J github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI= @@ -123,6 +131,8 @@ github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUc github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/schollz/progressbar/v3 v3.18.0 h1:uXdoHABRFmNIjUfte/Ex7WtuyVslrw2wVPQmCN62HpA= +github.com/schollz/progressbar/v3 v3.18.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec= github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= @@ -131,6 +141,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/vbauerster/mpb/v8 v8.10.1 h1:t/ZFv/NYgoBUy2LrmkD5Vc25r+JhoS4+gRkjVbolO2Y= +github.com/vbauerster/mpb/v8 v8.10.1/go.mod h1:+Ja4P92E3/CorSZgfDtK46D7AVbDqmBQRTmyTqPElo0= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= @@ -154,6 +166,10 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= +golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/ui/ui.go b/ui/ui.go index 2656d7e..52c32a4 100644 --- a/ui/ui.go +++ b/ui/ui.go @@ -2,6 +2,7 @@ package ui import ( "fmt" + "os" "sync" "time" @@ -74,8 +75,8 @@ func Init() { go func() { ui.program = tea.NewProgram( model, - tea.WithoutSignalHandler(), tea.WithInput(nil), + tea.WithOutput(os.Stderr), ) close(ui.ready) if _, err := ui.program.Run(); err != nil { From 479f1c45c241f53dd2f4f77f9ad53b576bca0d58 Mon Sep 17 00:00:00 2001 From: Nofre Date: Sat, 17 May 2025 18:30:01 +0200 Subject: [PATCH 2/6] chore(mod): added viper to project --- go.mod | 7 ------- go.sum | 16 ---------------- 2 files changed, 23 deletions(-) diff --git a/go.mod b/go.mod index cddb245..cb3e4e8 100644 --- a/go.mod +++ b/go.mod @@ -14,8 +14,6 @@ require ( require ( github.com/RoaringBitmap/roaring/v2 v2.4.5 // indirect - github.com/VividCortex/ewma v1.2.0 // indirect - github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/bits-and-blooms/bitset v1.22.0 // indirect github.com/blevesearch/bleve_index_api v1.2.8 // indirect @@ -50,21 +48,17 @@ require ( github.com/google/flatbuffers v25.2.10+incompatible // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v0.0.0-20171115153421-f7279a603ede // indirect - github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 // indirect github.com/klauspost/compress v1.18.0 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect - github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/mschoch/smat v0.2.0 // indirect github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/termenv v0.16.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect - github.com/schollz/progressbar/v3 v3.18.0 // indirect github.com/spf13/pflag v1.0.6 // indirect - github.com/vbauerster/mpb/v8 v8.10.1 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect go.etcd.io/bbolt v1.4.0 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect @@ -74,7 +68,6 @@ require ( golang.org/x/net v0.38.0 // indirect golang.org/x/sync v0.13.0 // indirect golang.org/x/sys v0.33.0 // indirect - golang.org/x/term v0.32.0 // indirect golang.org/x/text v0.23.0 // indirect google.golang.org/protobuf v1.36.6 // indirect ) diff --git a/go.sum b/go.sum index 2bac064..1bd3d90 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,5 @@ github.com/RoaringBitmap/roaring/v2 v2.4.5 h1:uGrrMreGjvAtTBobc0g5IrW1D5ldxDQYe2JW2gggRdg= github.com/RoaringBitmap/roaring/v2 v2.4.5/go.mod h1:FiJcsfkGje/nZBZgCu0ZxCPOKD/hVXDS2dXi7/eUFE0= -github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= -github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= -github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= -github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/adrg/xdg v0.5.3 h1:xRnxJXne7+oWDatRhR1JLnvuccuIeCoBu2rtuLqQB78= github.com/adrg/xdg v0.5.3/go.mod h1:nlTsY+NNiCBGCK2tpm09vRqfVzrc2fLmXGpBLF0zlTQ= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= @@ -97,8 +93,6 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/json-iterator/go v0.0.0-20171115153421-f7279a603ede h1:YrgBGwxMRK0Vq0WSCWFaZUnTsrA/PZE/xs1QZh+/edg= github.com/json-iterator/go v0.0.0-20171115153421-f7279a603ede/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 h1:qGQQKEcAR99REcMpsXCp3lJ03zYT1PkRd3kQGPn9GVg= -github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -113,8 +107,6 @@ github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2J github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= -github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI= @@ -131,8 +123,6 @@ github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUc github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/schollz/progressbar/v3 v3.18.0 h1:uXdoHABRFmNIjUfte/Ex7WtuyVslrw2wVPQmCN62HpA= -github.com/schollz/progressbar/v3 v3.18.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec= github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= @@ -141,8 +131,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/vbauerster/mpb/v8 v8.10.1 h1:t/ZFv/NYgoBUy2LrmkD5Vc25r+JhoS4+gRkjVbolO2Y= -github.com/vbauerster/mpb/v8 v8.10.1/go.mod h1:+Ja4P92E3/CorSZgfDtK46D7AVbDqmBQRTmyTqPElo0= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= @@ -164,12 +152,8 @@ golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= -golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= -golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From e54b70283d21d2c81cc4a4e97af7361c30dd1917 Mon Sep 17 00:00:00 2001 From: Nofre Date: Sat, 17 May 2025 21:20:25 +0200 Subject: [PATCH 3/6] feat(db): updated way to storing and indexing mans by versioning --- cmd/cli/apply.go | 2 + cmd/cli/search.go | 40 ++ cmd/cli/version.go | 1 + examples/simple/http_test.yaml | 2 +- internal/core/manifests/index/index.go | 3 + internal/core/manifests/interface.go | 8 +- internal/core/manifests/kinds/meta.go | 10 + .../core/manifests/kinds/servers/server.go | 4 +- .../core/manifests/kinds/services/service.go | 4 +- .../core/manifests/kinds/system/plan/plan.go | 4 +- .../core/manifests/kinds/tests/api/http.go | 3 +- .../core/manifests/kinds/tests/load/http.go | 4 +- .../core/manifests/kinds/values/values.go | 4 +- internal/core/manifests/loader/loader.go | 14 +- internal/core/store/db.go | 408 ++++++++++++++---- internal/core/store/helpers.go | 16 +- internal/core/store/index.go | 59 ++- 17 files changed, 458 insertions(+), 128 deletions(-) create mode 100644 cmd/cli/search.go diff --git a/cmd/cli/apply.go b/cmd/cli/apply.go index b6112b9..6244570 100644 --- a/cmd/cli/apply.go +++ b/cmd/cli/apply.go @@ -44,6 +44,8 @@ var applyCmd = &cobra.Command{ ui.Spinner(false) ui.Println("Manifests applied successfully") + + // => }, } diff --git a/cmd/cli/search.go b/cmd/cli/search.go new file mode 100644 index 0000000..e03d0f7 --- /dev/null +++ b/cmd/cli/search.go @@ -0,0 +1,40 @@ +package cli + +import "github.com/spf13/cobra" + +var searchCmd = &cobra.Command{ + Use: "search", + Short: "Search for manifests using filters", + SilenceUsage: true, + SilenceErrors: true, + Run: func(cmd *cobra.Command, args []string) {}, +} + +func init() { + // All + searchCmd.Flags().StringP("all", "a", "", "Get all manifests") + + // Exact match filters + searchCmd.Flags().StringP("name", "n", "", "Search manifest by exact name match") + searchCmd.Flags().StringP("namespace", "s", "default", "Search manifests by exact namespace)") + searchCmd.Flags().StringP("kind", "k", "", "Search manifests by exact kind (e.g., Server, HttpTest, HttpLoadTest)") + searchCmd.Flags().IntP("version", "v", 1, "Search manifests by exact version number") + + // Wildcard/partial match filters + searchCmd.Flags().StringP("wildcard", "w", "", "Search manifests using name wildcard pattern") + searchCmd.Flags().StringP("hash", "h", "", "Search manifests by hash prefix") + + // Dependency filters + searchCmd.Flags().StringSliceP("depends", "d", []string{}, "Search manifests by dependencies (comma separated)") + + // Date filters + searchCmd.Flags().String("created-after", "", "Search manifests created after date (YYYY-MM-DD)") + searchCmd.Flags().String("created-before", "", "Search manifests created before date") + searchCmd.Flags().String("updated-after", "", "Search manifests updated after date") + searchCmd.Flags().String("last-applied", "", "Search manifests by last applied date") + + // Creator filter + searchCmd.Flags().String("created-by", "", "Filter by creator username") + + rootCmd.AddCommand(searchCmd) +} diff --git a/cmd/cli/version.go b/cmd/cli/version.go index fd16177..c12d00d 100644 --- a/cmd/cli/version.go +++ b/cmd/cli/version.go @@ -2,6 +2,7 @@ package cli import ( "fmt" + "github.com/spf13/cobra" ) diff --git a/examples/simple/http_test.yaml b/examples/simple/http_test.yaml index a2d83cd..d0d78dc 100644 --- a/examples/simple/http_test.yaml +++ b/examples/simple/http_test.yaml @@ -34,7 +34,7 @@ spec: method: GET endpoint: /users/{id} expected: - code: 404 + code: 201 message: "User not found" dependsOn: diff --git a/internal/core/manifests/index/index.go b/internal/core/manifests/index/index.go index 1ddfb07..f4e5853 100644 --- a/internal/core/manifests/index/index.go +++ b/internal/core/manifests/index/index.go @@ -1,6 +1,7 @@ package index const ( + ID = "id" Version = "version" Kind = "kind" Name = "name" @@ -10,6 +11,8 @@ const ( const ( MetaHash = "meta.hash" + MetaVersion = "meta.version" + MetaIsCurrent = "meta.isCurrent" MetaCreatedAt = "meta.createdAt" MetaCreatedBy = "meta.createdBy" MetaUpdatedAt = "meta.updatedAt" diff --git a/internal/core/manifests/interface.go b/internal/core/manifests/interface.go index 61d75f9..7e72f8f 100644 --- a/internal/core/manifests/interface.go +++ b/internal/core/manifests/interface.go @@ -27,16 +27,13 @@ type Manifest interface { GetName() string GetNamespace() string Index() any + GetMeta() Meta } type Dependencies interface { GetDependsOn() []string } -type MetaTable interface { - GetMeta() Meta -} - type Meta interface { GetHash() string SetHash(hash string) @@ -45,6 +42,9 @@ type Meta interface { SetVersion(version uint8) IncVersion() + GetIsCurrent() bool + SetIsCurrent(isCurrent bool) + GetCreatedAt() time.Time SetCreatedAt(createdAt time.Time) diff --git a/internal/core/manifests/kinds/meta.go b/internal/core/manifests/kinds/meta.go index d79ece6..6fe19f0 100644 --- a/internal/core/manifests/kinds/meta.go +++ b/internal/core/manifests/kinds/meta.go @@ -21,6 +21,7 @@ func DefaultMeta() *Meta { return &Meta{ Hash: "", Version: 1, + IsCurrent: true, CreatedAt: time.Now(), CreatedBy: name, UpdatedAt: time.Now(), @@ -35,6 +36,7 @@ var _ manifests.Meta = (*Meta)(nil) type Meta struct { Hash string `yaml:"-" json:"hash"` Version uint8 `yaml:"-" json:"version"` + IsCurrent bool `yaml:"-" json:"isCurrent"` CreatedAt time.Time `yaml:"-" json:"createdAt"` CreatedBy string `yaml:"-" json:"createdBy"` UpdatedAt time.Time `yaml:"-" json:"updatedAt"` @@ -65,6 +67,14 @@ func (m *Meta) IncVersion() { } } +func (m *Meta) GetIsCurrent() bool { + return m.IsCurrent +} + +func (m *Meta) SetIsCurrent(isCurrent bool) { + m.IsCurrent = isCurrent +} + func (m *Meta) GetCreatedAt() time.Time { return m.CreatedAt } diff --git a/internal/core/manifests/kinds/servers/server.go b/internal/core/manifests/kinds/servers/server.go index 3c61d96..9981620 100644 --- a/internal/core/manifests/kinds/servers/server.go +++ b/internal/core/manifests/kinds/servers/server.go @@ -10,7 +10,6 @@ import ( var ( _ manifests.Manifest = (*Server)(nil) - _ manifests.MetaTable = (*Server)(nil) _ manifests.Defaultable = (*Server)(nil) _ manifests.Prepare = (*Server)(nil) ) @@ -44,12 +43,15 @@ func (s *Server) GetNamespace() string { func (s *Server) Index() any { return map[string]any{ + index.ID: s.GetID(), index.Version: float64(s.Version), index.Kind: s.Kind, index.Name: s.Name, index.Namespace: s.Namespace, index.MetaHash: s.Meta.Hash, + index.MetaVersion: float64(s.Meta.Version), + index.MetaIsCurrent: s.Meta.IsCurrent, index.MetaCreatedAt: s.Meta.CreatedAt.Format(time.RFC3339Nano), index.MetaCreatedBy: s.Meta.CreatedBy, index.MetaUpdatedAt: s.Meta.UpdatedAt.Format(time.RFC3339Nano), diff --git a/internal/core/manifests/kinds/services/service.go b/internal/core/manifests/kinds/services/service.go index 3d58079..7b0943a 100644 --- a/internal/core/manifests/kinds/services/service.go +++ b/internal/core/manifests/kinds/services/service.go @@ -11,7 +11,6 @@ import ( var ( _ manifests.Manifest = (*Service)(nil) _ manifests.Dependencies = (*Service)(nil) - _ manifests.MetaTable = (*Service)(nil) _ manifests.Defaultable = (*Service)(nil) _ manifests.Prepare = (*Service)(nil) ) @@ -49,6 +48,7 @@ func (s *Service) GetDependsOn() []string { func (s *Service) Index() any { return map[string]any{ + index.ID: s.GetID(), index.Version: float64(s.Version), index.Kind: s.Kind, index.Name: s.Name, @@ -56,6 +56,8 @@ func (s *Service) Index() any { index.DependsOn: s.DependsOn, index.MetaHash: s.Meta.Hash, + index.MetaVersion: float64(s.Meta.Version), + index.MetaIsCurrent: s.Meta.IsCurrent, index.MetaCreatedAt: s.Meta.CreatedAt.Format(time.RFC3339Nano), index.MetaCreatedBy: s.Meta.CreatedBy, index.MetaUpdatedAt: s.Meta.UpdatedAt.Format(time.RFC3339Nano), diff --git a/internal/core/manifests/kinds/system/plan/plan.go b/internal/core/manifests/kinds/system/plan/plan.go index 563c157..b62b38c 100644 --- a/internal/core/manifests/kinds/system/plan/plan.go +++ b/internal/core/manifests/kinds/system/plan/plan.go @@ -10,7 +10,6 @@ import ( var ( _ manifests.Manifest = (*Plan)(nil) - _ manifests.MetaTable = (*Plan)(nil) _ manifests.Defaultable = (*Plan)(nil) _ manifests.Prepare = (*Plan)(nil) ) @@ -59,12 +58,15 @@ func (p *Plan) GetNamespace() string { func (p *Plan) Index() any { return map[string]any{ + index.ID: p.GetID(), index.Version: float64(p.Version), index.Kind: p.Kind, index.Name: p.Name, index.Namespace: p.Namespace, index.MetaHash: p.Meta.Hash, + index.MetaVersion: float64(p.Meta.Version), + index.MetaIsCurrent: p.Meta.IsCurrent, index.MetaCreatedAt: p.Meta.CreatedAt.Format(time.RFC3339Nano), index.MetaCreatedBy: p.Meta.CreatedBy, index.MetaUpdatedAt: p.Meta.UpdatedAt.Format(time.RFC3339Nano), diff --git a/internal/core/manifests/kinds/tests/api/http.go b/internal/core/manifests/kinds/tests/api/http.go index 9f495f2..0018e20 100644 --- a/internal/core/manifests/kinds/tests/api/http.go +++ b/internal/core/manifests/kinds/tests/api/http.go @@ -15,7 +15,6 @@ import ( var ( _ manifests.Manifest = (*Http)(nil) _ manifests.Dependencies = (*Http)(nil) - _ manifests.MetaTable = (*Http)(nil) _ manifests.Defaultable = (*Http)(nil) _ manifests.Prepare = (*Http)(nil) ) @@ -61,6 +60,8 @@ func (h *Http) Index() any { index.DependsOn: h.DependsOn, index.MetaHash: h.Meta.Hash, + index.MetaVersion: float64(h.Meta.Version), + index.MetaIsCurrent: h.Meta.IsCurrent, index.MetaCreatedAt: h.Meta.CreatedAt.Format(time.RFC3339Nano), index.MetaCreatedBy: h.Meta.CreatedBy, index.MetaUpdatedAt: h.Meta.UpdatedAt.Format(time.RFC3339Nano), diff --git a/internal/core/manifests/kinds/tests/load/http.go b/internal/core/manifests/kinds/tests/load/http.go index 6534828..87a87e3 100644 --- a/internal/core/manifests/kinds/tests/load/http.go +++ b/internal/core/manifests/kinds/tests/load/http.go @@ -13,7 +13,6 @@ import ( var ( _ manifests.Manifest = (*Http)(nil) _ manifests.Dependencies = (*Http)(nil) - _ manifests.MetaTable = (*Http)(nil) _ manifests.Defaultable = (*Http)(nil) _ manifests.Prepare = (*Http)(nil) ) @@ -77,6 +76,7 @@ func (h *Http) GetNamespace() string { func (h *Http) Index() any { return map[string]any{ + index.ID: h.GetID(), index.Version: float64(h.Version), index.Kind: h.Kind, index.Name: h.Name, @@ -84,6 +84,8 @@ func (h *Http) Index() any { index.DependsOn: h.DependsOn, index.MetaHash: h.Meta.Hash, + index.MetaVersion: float64(h.Meta.Version), + index.MetaIsCurrent: h.Meta.IsCurrent, index.MetaCreatedAt: h.Meta.CreatedAt.Format(time.RFC3339Nano), index.MetaCreatedBy: h.Meta.CreatedBy, index.MetaUpdatedAt: h.Meta.UpdatedAt.Format(time.RFC3339Nano), diff --git a/internal/core/manifests/kinds/values/values.go b/internal/core/manifests/kinds/values/values.go index 22fe819..b9c281b 100644 --- a/internal/core/manifests/kinds/values/values.go +++ b/internal/core/manifests/kinds/values/values.go @@ -11,7 +11,6 @@ import ( var ( _ manifests.Manifest = (*Values)(nil) _ manifests.Defaultable = (*Values)(nil) - _ manifests.MetaTable = (*Values)(nil) _ manifests.Prepare = (*Values)(nil) ) @@ -47,12 +46,15 @@ func (v *Values) GetNamespace() string { func (v *Values) Index() any { return map[string]any{ + index.ID: v.GetID(), index.Version: float64(v.Version), index.Kind: v.Kind, index.Name: v.Name, index.Namespace: v.Namespace, index.MetaHash: v.Meta.Hash, + index.MetaVersion: float64(v.Meta.Version), + index.MetaIsCurrent: v.Meta.IsCurrent, index.MetaCreatedAt: v.Meta.CreatedAt.Format(time.RFC3339Nano), index.MetaCreatedBy: v.Meta.CreatedBy, index.MetaUpdatedAt: v.Meta.UpdatedAt.Format(time.RFC3339Nano), diff --git a/internal/core/manifests/loader/loader.go b/internal/core/manifests/loader/loader.go index 00a4a35..f3694e7 100644 --- a/internal/core/manifests/loader/loader.go +++ b/internal/core/manifests/loader/loader.go @@ -75,14 +75,12 @@ func LoadManifestsFromDir(dir string) ([]manifests.Manifest, error) { continue } - if metaTable, ok := m.(manifests.MetaTable); ok { - meta := metaTable.GetMeta() - meta.SetHash(fileHash) - meta.SetVersion(1) - now := time.Now() - meta.SetCreatedAt(now) - meta.SetUpdatedAt(now) - } + meta := m.GetMeta() + meta.SetHash(fileHash) + meta.SetVersion(1) + now := time.Now() + meta.SetCreatedAt(now) + meta.SetUpdatedAt(now) manifestsSet[manifestID] = struct{}{} manifestsList = append(manifestsList, m) diff --git a/internal/core/store/db.go b/internal/core/store/db.go index fef57cb..e3929b2 100644 --- a/internal/core/store/db.go +++ b/internal/core/store/db.go @@ -6,10 +6,11 @@ import ( "fmt" "os" "path/filepath" + "sort" + "strconv" + "strings" "time" - "github.com/apiqube/cli/ui" - "github.com/apiqube/cli/internal/core/manifests/index" "github.com/apiqube/cli/internal/core/manifests/parsing" bleceQuery "github.com/blevesearch/bleve/v2/search/query" @@ -21,7 +22,11 @@ import ( ) const ( - StorageDirPath = "ApiQube/Storage" + StorageDirPath = "ApiQube/storage" +) + +const ( + MaxManifestVersion = 10 ) type Storage struct { @@ -64,96 +69,148 @@ func NewStorage() (*Storage, error) { } func (s *Storage) SaveManifests(mans ...manifests.Manifest) error { - var err error - err = instance.db.Update(func(txn *badger.Txn) error { - var data []byte - + return instance.db.Update(func(txn *badger.Txn) error { for _, m := range mans { - data, err = json.Marshal(m) + currentVer, err := s.getCurrentVersion(txn, m.GetID()) if err != nil { - return fmt.Errorf("error marshalling manifest: %v", err) + return fmt.Errorf("version check failed: %v", err) } - if err = txn.Set(genManifestKey(m.GetID()), data); err != nil { + newVersion := currentVer + 1 + + meta := m.GetMeta() + meta.SetVersion(uint8(newVersion)) + meta.SetUpdatedAt(time.Now()) + meta.SetUpdatedBy("qube") + + versionedKey := genVersionedKey(m.GetID(), newVersion) + latestKey := genLatestKey(m.GetID()) + + data, err := json.Marshal(m) + if err != nil { + return fmt.Errorf("marshaling error: %v", err) + } + + if err = txn.Set(versionedKey, data); err != nil { + return err + } + if err = txn.Set(latestKey, versionedKey); err != nil { return err } - if err = s.index.Index(m.GetID(), m.Index()); err != nil { - ui.Errorf("Failed to index manifest %s: %v", m.GetID(), err) + if err = s.indexCurrentManifest(m, string(versionedKey), currentVer); err != nil { return err } - } + if newVersion > MaxManifestVersion { + if err = s.cleanupOldVersions(txn, m.GetID(), MaxManifestVersion); err != nil { + return err + } + } + } return nil }) - - return err } func (s *Storage) LoadManifests(ids ...string) ([]manifests.Manifest, error) { - var results []manifests.Manifest - var rErr error + if len(ids) == 0 { + return nil, fmt.Errorf("empty IDs list") + } - err := instance.db.View(func(txn *badger.Txn) error { - var item *badger.Item - var err error + var ( + results []manifests.Manifest + errs error + ) + err := instance.db.View(func(txn *badger.Txn) error { for _, id := range ids { - item, err = txn.Get(genManifestKey(id)) - if errors.Is(err, badger.ErrKeyNotFound) { - rErr = errors.Join(rErr, fmt.Errorf("manifest %s not found", id)) + item, err := txn.Get(genLatestKey(id)) + if err != nil { + if errors.Is(err, badger.ErrKeyNotFound) { + errs = errors.Join(errs, fmt.Errorf("manifest %q not found", id)) + continue + } + errs = errors.Join(errs, fmt.Errorf("failed to get latest pointer for %q: %w", id, err)) continue - } else if err != nil { - rErr = errors.Join(rErr, err) + } + + var latestKey []byte + if err = item.Value(func(val []byte) error { + latestKey = append([]byte{}, val...) + return nil + }); err != nil { + errs = errors.Join(errs, fmt.Errorf("failed to read latest key for %q: %w", id, err)) continue } - var man manifests.Manifest + item, err = txn.Get(latestKey) + if err != nil { + errs = errors.Join(errs, fmt.Errorf("failed to get manifest data for %q: %w", id, err)) + continue + } + var manifest manifests.Manifest if err = item.Value(func(data []byte) error { - if man, err = parsing.ParseManifestAsJSON(data); err == nil { - results = append(results, man) - } - + manifest, err = parsing.ParseManifestAsJSON(data) return err }); err != nil { - rErr = errors.Join(rErr, err) + errs = errors.Join(errs, fmt.Errorf("parsing failed for %q: %w", id, err)) + continue } + + results = append(results, manifest) } + if len(results) == 0 && errs != nil { + return fmt.Errorf("no manifests loaded: %w", errs) + } return nil }) + if err != nil { + return results, fmt.Errorf("transaction failed: %w", errors.Join(err, errs)) + } - return results, errors.Join(rErr, err) + return results, errs } func (s *Storage) LoadManifest(id string) (manifests.Manifest, error) { - var result manifests.Manifest - var rErr error + var manifest manifests.Manifest err := instance.db.View(func(txn *badger.Txn) error { - item, err := txn.Get(genManifestKey(id)) - if errors.Is(err, badger.ErrKeyNotFound) { - rErr = errors.Join(rErr, fmt.Errorf("manifest %s not found", id)) + item, err := txn.Get(genLatestKey(id)) + if err != nil { + return fmt.Errorf("failed to get latest version pointer: %w", err) + } + + var versionedKey []byte + if err := item.Value(func(val []byte) error { + versionedKey = make([]byte, len(val)) + copy(versionedKey, val) return nil - } else if err != nil { - rErr = errors.Join(rErr, err) + }); err != nil { + return fmt.Errorf("failed to read version key: %w", err) } - if err = item.Value(func(data []byte) error { - if result, err = parsing.ParseManifestAsJSON(data); err != nil { - return err - } + item, err = txn.Get(versionedKey) + if err != nil { + return fmt.Errorf("failed to get manifest data: %w", err) + } - return nil + if err = item.Value(func(data []byte) error { + var parseErr error + manifest, parseErr = parsing.ParseManifestAsJSON(data) + return parseErr }); err != nil { - rErr = errors.Join(rErr, err) + return fmt.Errorf("parsing failed: %w", err) } return nil }) + if err != nil { + return nil, fmt.Errorf("failed to load manifest %q: %w", id, err) + } - return result, errors.Join(rErr, err) + return manifest, nil } func (s *Storage) FindManifestsByKind(kind string) ([]manifests.Manifest, error) { @@ -167,7 +224,7 @@ func (s *Storage) FindManifestsByKind(kind string) ([]manifests.Manifest, error) return nil, fmt.Errorf("error searching manifests: %v", err) } - return s.parseSearResults(searchResults) + return s.parseSearchResults(searchResults) } func (s *Storage) FindManifestsByVersion(lowVersion, heightVersion uint8) ([]manifests.Manifest, error) { @@ -183,7 +240,7 @@ func (s *Storage) FindManifestsByVersion(lowVersion, heightVersion uint8) ([]man return nil, fmt.Errorf("error searching manifests: %v", err) } - return s.parseSearResults(searchResults) + return s.parseSearchResults(searchResults) } func (s *Storage) FindManifestByName(name string) (manifests.Manifest, error) { @@ -198,7 +255,7 @@ func (s *Storage) FindManifestByName(name string) (manifests.Manifest, error) { return nil, fmt.Errorf("search failed: %w", err) } - results, err := s.parseSearResults(searchResults) + results, err := s.parseSearchResults(searchResults) if err != nil { return nil, err } @@ -221,7 +278,7 @@ func (s *Storage) FindManifestsByNameWildcard(namePattern string) ([]manifests.M return nil, fmt.Errorf("search failed: %w", err) } - return s.parseSearResults(searchResults) + return s.parseSearchResults(searchResults) } func (s *Storage) FindManifestsByNamespace(namespace string) ([]manifests.Manifest, error) { @@ -235,7 +292,7 @@ func (s *Storage) FindManifestsByNamespace(namespace string) ([]manifests.Manife return nil, fmt.Errorf("search failed: %w", err) } - return s.parseSearResults(searchResults) + return s.parseSearchResults(searchResults) } func (s *Storage) FindManifestByDependencies(dependencies []string, requireAll bool) ([]manifests.Manifest, error) { @@ -266,7 +323,7 @@ func (s *Storage) FindManifestByDependencies(dependencies []string, requireAll b return nil, fmt.Errorf("search failed: %w", err) } - return s.parseSearResults(searchResults) + return s.parseSearchResults(searchResults) } func (s *Storage) FindManifestByHash(hash string) (manifests.Manifest, error) { @@ -281,7 +338,7 @@ func (s *Storage) FindManifestByHash(hash string) (manifests.Manifest, error) { return nil, fmt.Errorf("search failed: %w", err) } - results, err := s.parseSearResults(searchResults) + results, err := s.parseSearchResults(searchResults) if err != nil { return nil, err } @@ -304,7 +361,7 @@ func (s *Storage) FindManifestsByCreatedAtRange(start, end time.Time) ([]manifes return nil, fmt.Errorf("error searching manifests: %v", err) } - return s.parseSearResults(searchResults) + return s.parseSearchResults(searchResults) } func (s *Storage) FindManifestsByCreatedBy(createdBy string) ([]manifests.Manifest, error) { @@ -318,7 +375,7 @@ func (s *Storage) FindManifestsByCreatedBy(createdBy string) ([]manifests.Manife return nil, fmt.Errorf("search failed: %w", err) } - return s.parseSearResults(searchResults) + return s.parseSearchResults(searchResults) } func (s *Storage) FindManifestsByUpdatedAtRange(start, end time.Time) ([]manifests.Manifest, error) { @@ -332,7 +389,7 @@ func (s *Storage) FindManifestsByUpdatedAtRange(start, end time.Time) ([]manifes return nil, fmt.Errorf("error searching manifests: %v", err) } - return s.parseSearResults(searchResults) + return s.parseSearchResults(searchResults) } func (s *Storage) FindManifestsByUpdatedBy(updatedBy string) ([]manifests.Manifest, error) { @@ -346,7 +403,7 @@ func (s *Storage) FindManifestsByUpdatedBy(updatedBy string) ([]manifests.Manife return nil, fmt.Errorf("search failed: %w", err) } - return s.parseSearResults(searchResults) + return s.parseSearchResults(searchResults) } func (s *Storage) FindManifestsByUsedBy(usedBy string) ([]manifests.Manifest, error) { @@ -360,7 +417,7 @@ func (s *Storage) FindManifestsByUsedBy(usedBy string) ([]manifests.Manifest, er return nil, fmt.Errorf("search failed: %w", err) } - return s.parseSearResults(searchResults) + return s.parseSearchResults(searchResults) } func (s *Storage) FindManifestsByLastAppliedRange(start, end time.Time) ([]manifests.Manifest, error) { @@ -374,43 +431,220 @@ func (s *Storage) FindManifestsByLastAppliedRange(start, end time.Time) ([]manif return nil, fmt.Errorf("error searching manifests: %v", err) } - return s.parseSearResults(searchResults) + return s.parseSearchResults(searchResults) } -func (s *Storage) parseSearResults(searchResults *bleve.SearchResult) ([]manifests.Manifest, error) { - var results []manifests.Manifest - var rErr error +func (s *Storage) parseSearchResults(searchResults *bleve.SearchResult) ([]manifests.Manifest, error) { + if searchResults == nil || searchResults.Total == 0 { + return nil, nil + } - if searchResults.Total > 0 { - for _, hit := range searchResults.Hits { - var man manifests.Manifest + var ( + manifestsList = make([]manifests.Manifest, 0, len(searchResults.Hits)) + errorsList []error + ) - err := s.db.View(func(txn *badger.Txn) error { - var item *badger.Item - var err error + for _, hit := range searchResults.Hits { + manifest, err := s.loadManifestByID(hit.ID) + if err != nil { + errorsList = append(errorsList, fmt.Errorf("id %q: %w", hit.ID, err)) + continue + } - item, err = txn.Get(genManifestKey(hit.ID)) - if err != nil && errors.Is(err, badger.ErrKeyNotFound) { - rErr = errors.Join(rErr, fmt.Errorf("could not find manifest by ID: %v", hit.ID)) - } else if err != nil { - rErr = errors.Join(rErr, fmt.Errorf("failed to load manifest by ID: %v", hit.ID)) - } + if manifest != nil { + manifestsList = append(manifestsList, manifest) + } + } - return item.Value(func(val []byte) error { - man, err = parsing.ParseManifest(parsing.JSONMethod, val) - if err != nil { - return err - } - return nil - }) - }) - if err != nil { - rErr = errors.Join(rErr, err) - } + return manifestsList, errors.Join(errorsList...) +} - results = append(results, man) +func (s *Storage) loadManifestByID(id string) (manifests.Manifest, error) { + var manifest manifests.Manifest + + err := instance.db.View(func(txn *badger.Txn) error { + item, err := txn.Get([]byte(id)) + if err != nil { + return err } + + return item.Value(func(data []byte) error { + var parseErr error + manifest, parseErr = parsing.ParseManifest(parsing.JSONMethod, data) + return parseErr + }) + }) + + switch { + case errors.Is(err, badger.ErrKeyNotFound): + return nil, fmt.Errorf("not found") + case err != nil: + return nil, fmt.Errorf("storage error: %w", err) + default: + return manifest, nil + } +} + +func (s *Storage) getCurrentVersion(txn *badger.Txn, id string) (int, error) { + item, err := txn.Get(genLatestKey(id)) + if errors.Is(err, badger.ErrKeyNotFound) { + return 0, nil + } + if err != nil { + return 0, err } - return results, rErr + val, err := item.ValueCopy(nil) + if err != nil { + return 0, err + } + + valStr := string(val) + parts := strings.Split(valStr, "@v") + if len(parts) != 2 { + return 0, fmt.Errorf("invalid versioned key format") + } + + return strconv.Atoi(parts[1]) +} + +func (s *Storage) indexCurrentManifest(m manifests.Manifest, newKey string, oldVersion int) error { + if oldVersion > 0 { + oldKey := genVersionedKey(m.GetID(), oldVersion) + if err := s.index.Delete(string(oldKey)); err != nil { + return fmt.Errorf("could not delete manifest: %s old version %v: %v", m.GetID(), oldVersion, err) + } + } + + m.GetMeta().SetIsCurrent(true) + + if err := s.index.Index(newKey, m.Index()); err != nil { + return fmt.Errorf("could not index manifest: %s old version %v: %v", m.GetID(), oldVersion, err) + } + + return nil +} + +func (s *Storage) cleanupOldVersions(txn *badger.Txn, id string, keepVersions int) error { + versions, err := s.getAllVersions(txn, id) + if err != nil { + return err + } + + sort.Ints(versions) + + for i := 0; i < len(versions)-keepVersions; i++ { + key := genVersionedKey(id, versions[i]) + + if err = txn.Delete(key); err != nil { + return fmt.Errorf("could not delete manifest: %s old version %v: %v", id, versions[i], err) + } + + if err = s.index.Delete(string(key)); err != nil { + return fmt.Errorf("could not delete manifest: %s old version %v: %v", id, versions[i], err) + } + } + + if versions[len(versions)-1] > MaxManifestVersion { + return s.resetVersions(txn, id) + } + + return nil +} + +func (s *Storage) resetVersions(txn *badger.Txn, id string) error { + latestKey := genLatestKey(id) + item, err := txn.Get(latestKey) + if err != nil { + return fmt.Errorf("failed to get latest version: %w", err) + } + + latestVal, err := item.ValueCopy(nil) + if err != nil { + return fmt.Errorf("failed to read latest value: %w", err) + } + + item, err = txn.Get(latestVal) + if err != nil { + return fmt.Errorf("failed to get manifest data: %w", err) + } + + manifestData, err := item.ValueCopy(nil) + if err != nil { + return fmt.Errorf("failed to copy manifest data: %w", err) + } + + manifest, err := parsing.ParseManifest(parsing.JSONMethod, manifestData) + if err != nil { + return fmt.Errorf("parsing failed: %w", err) + } + + meta := manifest.GetMeta() + meta.SetVersion(1) + meta.SetUpdatedAt(time.Now()) + meta.SetUpdatedBy("qube") + meta.SetIsCurrent(true) + + newData, err := json.Marshal(manifest) + if err != nil { + return fmt.Errorf("marshaling failed: %w", err) + } + + newKey := genVersionedKey(id, 1) + if err = txn.Set(newKey, newData); err != nil { + return fmt.Errorf("failed to save new version: %w", err) + } + + if err = txn.Set(latestKey, newKey); err != nil { + return fmt.Errorf("failed to update latest pointer: %w", err) + } + + if err = s.cleanupIndexForReset(id, newKey); err != nil { + return fmt.Errorf("index cleanup failed: %w", err) + } + + return s.index.Index(string(newKey), manifest.Index()) +} + +func (s *Storage) cleanupIndexForReset(id string, newKey []byte) error { + query := bleve.NewTermQuery(id) + searchReq := bleve.NewSearchRequest(query) + searchReq.Fields = []string{"*"} + + searchResults, err := s.index.Search(searchReq) + if err != nil { + return err + } + + var delErrors error + for _, hit := range searchResults.Hits { + if hit.ID != string(newKey) { + if err := s.index.Delete(hit.ID); err != nil { + delErrors = errors.Join(delErrors, + fmt.Errorf("failed to delete %s: %w", hit.ID, err)) + } + } + } + return delErrors +} + +func (s *Storage) getAllVersions(txn *badger.Txn, id string) ([]int, error) { + var versions []int + it := txn.NewIterator(badger.DefaultIteratorOptions) + defer it.Close() + + prefix := []byte(id + "@v") + for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() { + key := string(it.Item().Key()) + parts := strings.Split(key, "@v") + if len(parts) != 2 { + continue + } + ver, err := strconv.Atoi(parts[1]) + if err != nil { + continue + } + versions = append(versions, ver) + } + return versions, nil } diff --git a/internal/core/store/helpers.go b/internal/core/store/helpers.go index f7cf4b6..a0bb553 100644 --- a/internal/core/store/helpers.go +++ b/internal/core/store/helpers.go @@ -1,5 +1,17 @@ package store -func genManifestKey(id string) []byte { - return []byte(id) +import ( + "fmt" +) + +const ( + manifestsLatestKey = "latest/" +) + +func genLatestKey(id string) []byte { + return []byte(fmt.Sprintf("%s%s", manifestsLatestKey, id)) +} + +func genVersionedKey(id string, version int) []byte { + return []byte(fmt.Sprintf("%s@v%d", id, version)) } diff --git a/internal/core/store/index.go b/internal/core/store/index.go index 065a55f..121eb2d 100644 --- a/internal/core/store/index.go +++ b/internal/core/store/index.go @@ -7,36 +7,55 @@ import ( ) func buildBleveMapping() *mapping.IndexMappingImpl { + indexMapping := bleve.NewIndexMapping() + indexMapping.DefaultAnalyzer = "standard" + manifestMapping := bleve.NewDocumentMapping() - // Main - manifestMapping.AddFieldMappingsAt(index.Version, bleve.NewNumericFieldMapping()) + idMapping := bleve.NewTextFieldMapping() + idMapping.Analyzer = "keyword" + idMapping.Store = true + manifestMapping.AddFieldMappingsAt(index.ID, idMapping) - kindMapping := bleve.NewTextFieldMapping() - kindMapping.Analyzer = "keyword" - manifestMapping.AddFieldMappingsAt(index.Kind, kindMapping) + manifestMapping.AddFieldMappingsAt(index.Version, bleve.NewNumericFieldMapping()) + manifestMapping.AddFieldMappingsAt(index.MetaVersion, bleve.NewNumericFieldMapping()) + + exactMatchFields := []string{ + index.Kind, + index.Namespace, + index.DependsOn, + index.MetaCreatedBy, + index.MetaUpdatedBy, + index.MetaUsedBy, + } + + for _, field := range exactMatchFields { + fm := bleve.NewTextFieldMapping() + fm.Analyzer = "keyword" + manifestMapping.AddFieldMappingsAt(field, fm) + } nameMapping := bleve.NewTextFieldMapping() + nameMapping.Analyzer = "standard" manifestMapping.AddFieldMappingsAt(index.Name, nameMapping) - dependsMapping := bleve.NewTextFieldMapping() - dependsMapping.Analyzer = "keyword" - manifestMapping.AddFieldMappingsAt(index.DependsOn, dependsMapping) + hashMapping := bleve.NewTextFieldMapping() + hashMapping.Analyzer = "keyword" + hashMapping.Store = true + manifestMapping.AddFieldMappingsAt(index.MetaHash, hashMapping) - namespaceMapping := bleve.NewTextFieldMapping() - namespaceMapping.Analyzer = "keyword" - manifestMapping.AddFieldMappingsAt(index.Namespace, namespaceMapping) + dateTimeMapping := bleve.NewDateTimeFieldMapping() - // Meta - manifestMapping.AddFieldMappingsAt(index.MetaHash, bleve.NewTextFieldMapping()) - manifestMapping.AddFieldMappingsAt(index.MetaCreatedAt, bleve.NewDateTimeFieldMapping()) - manifestMapping.AddFieldMappingsAt(index.MetaCreatedBy, bleve.NewTextFieldMapping()) - manifestMapping.AddFieldMappingsAt(index.MetaUpdatedAt, bleve.NewDateTimeFieldMapping()) - manifestMapping.AddFieldMappingsAt(index.MetaUpdatedBy, bleve.NewTextFieldMapping()) - manifestMapping.AddFieldMappingsAt(index.MetaUsedBy, bleve.NewTextFieldMapping()) - manifestMapping.AddFieldMappingsAt(index.MetaLastApplied, bleve.NewDateTimeFieldMapping()) + dateFields := []string{ + index.MetaCreatedAt, + index.MetaUpdatedAt, + index.MetaLastApplied, + } + + for _, field := range dateFields { + manifestMapping.AddFieldMappingsAt(field, dateTimeMapping) + } - indexMapping := bleve.NewIndexMapping() indexMapping.DefaultMapping = manifestMapping return indexMapping From f291d2ca01806cbc6278c1df012d8aac35f84dcb Mon Sep 17 00:00:00 2001 From: Nofre Date: Sun, 18 May 2025 00:25:30 +0200 Subject: [PATCH 4/6] feat(cmd): added new cli command (cleanup and rollback) --- cmd/cli/apply.go | 16 ++-------- cmd/cli/cleanup.go | 78 +++++++++++++++++++++++++++++++++++++++++++++ cmd/cli/rollback.go | 75 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 156 insertions(+), 13 deletions(-) create mode 100644 cmd/cli/cleanup.go create mode 100644 cmd/cli/rollback.go diff --git a/cmd/cli/apply.go b/cmd/cli/apply.go index 6244570..7217b71 100644 --- a/cmd/cli/apply.go +++ b/cmd/cli/apply.go @@ -27,16 +27,17 @@ var applyCmd = &cobra.Command{ ui.Printf("Applying manifests from: %s", file) ui.Spinner(true, "Loading manifests") - mans, err := loader.LoadManifestsFromDir(file) + loadedMans, _, err := loader.LoadManifests(file) if err != nil { ui.Spinner(false) ui.Errorf("Failed to load manifests: %s", err.Error()) return } + ui.Spinner(false) ui.Spinner(true, "Saving manifests...") - if err := store.SaveManifests(mans...); err != nil { + if err := store.SaveManifests(loadedMans...); err != nil { ui.Error("Failed to save manifests: " + err.Error()) ui.Spinner(false) return @@ -44,16 +45,5 @@ var applyCmd = &cobra.Command{ ui.Spinner(false) ui.Println("Manifests applied successfully") - - // => }, } - -/*type Logger struct { - Writer io.Writer - Style lipgloss.Style -} - -func (l *Logger) Log(msg string) { - fmt.Fprintln(l.Writer, l.Style.Render(msg)) -}*/ diff --git a/cmd/cli/cleanup.go b/cmd/cli/cleanup.go new file mode 100644 index 0000000..aef8f77 --- /dev/null +++ b/cmd/cli/cleanup.go @@ -0,0 +1,78 @@ +package cli + +import ( + "fmt" + + "github.com/apiqube/cli/internal/core/store" + "github.com/apiqube/cli/ui" + "github.com/spf13/cobra" +) + +const keepVersionDefault = 5 + +var cleanupCmd = &cobra.Command{ + Use: "cleanup [ID]", + Short: "Cleanup old manifest versions by its id", + Long: fmt.Sprintf("Delete all versions of the manifest,"+ + "\nleaving only the latest specified."+ + "\nBy default, the last keep amount is %d", keepVersionDefault), + SilenceErrors: true, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + opts, err := parseCleanUpFlags(cmd, args) + if err != nil { + ui.Errorf("Failed to parse provided values: %v", err) + return err + } + + keep := opts.Keep + if keep <= 0 { + keep = keepVersionDefault + } + + ui.Spinner(true, "Cleaning up...") + if err = store.CleanupOldVersions(opts.ManifestID, keep); err != nil { + ui.Spinner(false) + ui.Errorf("Failed to cleanup old versions: %v", err) + } + + ui.Spinner(false) + ui.Successf("Successfully cleaned up %v to last %d versions", opts.ManifestID, keep) + return nil + }, +} + +func init() { + rootCmd.AddCommand(cleanupCmd) + cleanupCmd.Flags().IntP("keep", "k", keepVersionDefault, "Number of last versions to keep") +} + +type CleanUpOptions struct { + ManifestID string + Keep int +} + +func parseCleanUpFlags(cmd *cobra.Command, args []string) (*CleanUpOptions, error) { + opts := &CleanUpOptions{} + + if len(args) == 0 { + return nil, fmt.Errorf("manifest ID is required") + } + + opts.ManifestID = args[0] + var err error + var keep int + + if cmd.Flags().Changed("keep") { + keep, err = cmd.Flags().GetInt("keep") + if err != nil { + return nil, fmt.Errorf("invalid keep value: %w", err) + } + if keep < 1 { + return nil, fmt.Errorf("keep value must be positive") + } + opts.Keep = keep + } + + return opts, nil +} diff --git a/cmd/cli/rollback.go b/cmd/cli/rollback.go new file mode 100644 index 0000000..0f315da --- /dev/null +++ b/cmd/cli/rollback.go @@ -0,0 +1,75 @@ +package cli + +import ( + "fmt" + + "github.com/apiqube/cli/internal/core/store" + "github.com/apiqube/cli/ui" + "github.com/spf13/cobra" +) + +var rollbackCmd = &cobra.Command{ + Use: "rollback [ID]", + Short: "Rollback to previous manifest version", + Long: fmt.Sprint("Rollback to specific version of manifest." + + "\nIf version is not specified, rolls back to previous one."), + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + opts, err := parseRollbackFlags(cmd, args) + if err != nil { + ui.Errorf("Failed to parse provided values: %v", err) + return + } + + targetVersion := opts.Version + if targetVersion <= 0 { + targetVersion = 1 + } + + ui.Spinner(true, "Rolling back") + if err = store.Rollback(opts.ManifestID, targetVersion); err != nil { + ui.Spinner(false, "Failed to rollback") + ui.Errorf("Error rolling back to previous version: %s", err) + return + } + + ui.Spinner(false) + ui.Successf("Successfully rolled back %s to version %d\n", opts.ManifestID, targetVersion) + }, +} + +func init() { + rollbackCmd.Flags().IntP("version", "v", 0, "Target version number (defaults to previous version)") + + rootCmd.AddCommand(rollbackCmd) +} + +type RollbackOptions struct { + ManifestID string + Version int +} + +func parseRollbackFlags(cmd *cobra.Command, args []string) (*RollbackOptions, error) { + opts := &RollbackOptions{} + + if len(args) == 0 { + return nil, fmt.Errorf("manifest ID is required") + } + + opts.ManifestID = args[0] + var err error + var ver int + + if cmd.Flags().Changed("version") { + ver, err = cmd.Flags().GetInt("version") + if err != nil { + return nil, fmt.Errorf("invalid version: %w", err) + } + if ver < 1 { + return nil, fmt.Errorf("version must be positive") + } + opts.Version = ver + } + + return opts, nil +} From a54f92268d8f2aca3ae9b6c7eb2615c3eac53138 Mon Sep 17 00:00:00 2001 From: Nofre Date: Sun, 18 May 2025 00:26:16 +0200 Subject: [PATCH 5/6] refactor: refactoring and updates in loading manifests logic, fixing issues --- internal/core/manifests/hash/hash.go | 8 + internal/core/manifests/loader/loader.go | 181 ++++++++++++++++------- internal/core/store/db.go | 66 ++++++++- internal/core/store/independ.go | 16 ++ ui/styles.go | 4 +- 5 files changed, 221 insertions(+), 54 deletions(-) diff --git a/internal/core/manifests/hash/hash.go b/internal/core/manifests/hash/hash.go index c58373d..75c1446 100644 --- a/internal/core/manifests/hash/hash.go +++ b/internal/core/manifests/hash/hash.go @@ -25,3 +25,11 @@ func CalculateHashWithPath(filePath string, content []byte) (string, error) { return hash, nil } + +func CalculateHashWithContent(content []byte) (string, error) { + hasher := sha256.New() + hasher.Write(content) + hash := hex.EncodeToString(hasher.Sum(nil)) + + return hash, nil +} diff --git a/internal/core/manifests/loader/loader.go b/internal/core/manifests/loader/loader.go index f3694e7..93588a1 100644 --- a/internal/core/manifests/loader/loader.go +++ b/internal/core/manifests/loader/loader.go @@ -4,9 +4,12 @@ import ( "fmt" "os" "path/filepath" + "sort" "strings" "time" + "gopkg.in/yaml.v3" + "github.com/apiqube/cli/internal/core/manifests/hash" "github.com/apiqube/cli/internal/core/manifests/parsing" "github.com/apiqube/cli/internal/core/store" @@ -15,88 +18,123 @@ import ( "github.com/apiqube/cli/internal/core/manifests" ) -func LoadManifestsFromDir(dir string) ([]manifests.Manifest, error) { - files, err := os.ReadDir(dir) +func LoadManifests(path string) (new []manifests.Manifest, cached []manifests.Manifest, err error) { + if isYAMLFile(path) { + return processSingleFile(path) + } + + files, err := os.ReadDir(path) if err != nil { - return nil, fmt.Errorf("failed to read directory: %w", err) + return nil, nil, fmt.Errorf("failed to read directory: %w", err) } - var ( - manifestsList []manifests.Manifest - manifestsSet = make(map[string]struct{}) - newCounter int - ) + manifestsSet := make(map[string]struct{}) + newCount, cachedCount := 0, 0 for _, file := range files { - if file.IsDir() || (!strings.HasSuffix(file.Name(), ".yaml") && !strings.HasSuffix(file.Name(), ".yml")) { + if file.IsDir() || !isYAMLFile(file.Name()) { continue } - var content []byte - filePath := filepath.Join(dir, file.Name()) - content, err = os.ReadFile(filePath) + filePath := filepath.Join(path, file.Name()) + + var fileNew, fileCached []manifests.Manifest + fileNew, fileCached, err = processFile(filePath, manifestsSet) if err != nil { - return nil, fmt.Errorf("error reading file %s: %w", filePath, err) + return nil, nil, err } - var fileHash string - fileHash, err = hash.CalculateHashWithPath(filePath, content) + new = append(new, fileNew...) + cached = append(cached, fileCached...) + newCount += len(fileNew) + cachedCount += len(fileCached) + } + + logResults(newCount, cachedCount) + return new, cached, nil +} + +func isYAMLFile(path string) bool { + return strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml") +} + +func processSingleFile(filePath string) (new []manifests.Manifest, cached []manifests.Manifest, err error) { + return processFile(filePath, make(map[string]struct{})) +} + +func processFile(filePath string, manifestsSet map[string]struct{}) (new []manifests.Manifest, cached []manifests.Manifest, err error) { + content, err := os.ReadFile(filePath) + if err != nil { + return nil, nil, fmt.Errorf("error reading file %s: %w", filePath, err) + } + + parsedManifests, err := parsing.ParseManifestsAsYAML(content) + if err != nil { + return nil, nil, fmt.Errorf("in file %s: %w", filepath.Base(filePath), err) + } + + now := time.Now() + var newManifests, cachedManifests []manifests.Manifest + + for _, m := range parsedManifests { + manifestID := m.GetID() + + var normalized []byte + normalized, err = normalizeYAML(m) if err != nil { - return nil, fmt.Errorf("failed to calculate hash for %s: %w", filePath, err) + return nil, nil, fmt.Errorf("failed to normalize manifest %s: %w", manifestID, err) + } + var manifestHash string + manifestHash, err = hash.CalculateHashWithContent(normalized) + if err != nil { + return nil, nil, fmt.Errorf("failed to calculate hash for manifest %s: %w", manifestID, err) } var existingManifest manifests.Manifest - existingManifest, err = store.FindManifestByHash(fileHash) - if err != nil && !strings.Contains(err.Error(), "no matching manifest found") { - return nil, fmt.Errorf("failed to check manifest existence: %w", err) + existingManifest, err = store.FindManifestByHash(manifestHash) + if err != nil && !isNotFoundError(err) { + return nil, nil, fmt.Errorf("failed to check manifest existence: %w", err) } if existingManifest != nil { if _, exists := manifestsSet[existingManifest.GetID()]; !exists { manifestsSet[existingManifest.GetID()] = struct{}{} - manifestsList = append(manifestsList, existingManifest) + ui.Infof("Manifest %s unchanged (%s) - using cached version", + existingManifest.GetID(), shortHash(manifestHash)) + cachedManifests = append(cachedManifests, existingManifest) } - - ui.Infof("Manifest file %s unchanged (%s) - using cached version", file.Name(), shortHash(fileHash)) continue } - var parsedManifests []manifests.Manifest - parsedManifests, err = parsing.ParseManifestsAsYAML(content) - if err != nil { - return nil, fmt.Errorf("in file %s: %w", file.Name(), err) + if _, exists := manifestsSet[manifestID]; exists { + ui.Warningf("Duplicate manifest ID: %s (from %s)", manifestID, filepath.Base(filePath)) + continue } - for _, m := range parsedManifests { - manifestID := m.GetID() + meta := m.GetMeta() + meta.SetHash(manifestHash) + meta.SetVersion(1) + meta.SetCreatedAt(now) + meta.SetUpdatedAt(now) - if _, exists := manifestsSet[manifestID]; exists { - ui.Warningf("Duplicate manifest ID: %s (from %s)", manifestID, file.Name()) - continue - } - - meta := m.GetMeta() - meta.SetHash(fileHash) - meta.SetVersion(1) - now := time.Now() - meta.SetCreatedAt(now) - meta.SetUpdatedAt(now) + manifestsSet[manifestID] = struct{}{} + newManifests = append(newManifests, m) + ui.Successf("New manifest added: %s (h: %s)", manifestID, shortHash(manifestHash)) + } - manifestsSet[manifestID] = struct{}{} - manifestsList = append(manifestsList, m) - newCounter++ + return newManifests, cachedManifests, nil +} - ui.Successf("New manifest added: %s (h: %s)", manifestID, shortHash(fileHash)) - } +func logResults(cachedCount, newCount int) { + if cachedCount > 0 { + ui.Infof("Loaded %d cached manifests", cachedCount) } - - if newCounter == 0 { - ui.Info("No new manifests found in directory") - } else { - ui.Infof("Loaded %d new manifests", newCounter) + if newCount > 0 { + ui.Infof("Loaded %d new manifests", newCount) + } + if cachedCount == 0 && newCount == 0 { + ui.Info("No manifests found") } - - return manifestsList, nil } func shortHash(fullHash string) string { @@ -105,3 +143,44 @@ func shortHash(fullHash string) string { } return fullHash } + +func normalizeYAML(m manifests.Manifest) ([]byte, error) { + data, err := yaml.Marshal(m) + if err != nil { + return nil, err + } + + var raw map[string]interface{} + if err = yaml.Unmarshal(data, &raw); err != nil { + return nil, err + } + + sorted := sortMapKeys(raw) + + return yaml.Marshal(sorted) +} + +func sortMapKeys(m map[string]interface{}) map[string]interface{} { + res := make(map[string]interface{}) + keys := make([]string, 0, len(m)) + + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + + for _, k := range keys { + if nested, ok := m[k].(map[string]interface{}); ok { + res[k] = sortMapKeys(nested) + } else { + res[k] = m[k] + } + } + + return res +} + +func isNotFoundError(err error) bool { + return err != nil && (strings.Contains(err.Error(), "no matching manifest found") || + strings.Contains(err.Error(), "not found")) +} diff --git a/internal/core/store/db.go b/internal/core/store/db.go index e3929b2..33ee985 100644 --- a/internal/core/store/db.go +++ b/internal/core/store/db.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + "math" "os" "path/filepath" "sort" @@ -26,7 +27,7 @@ const ( ) const ( - MaxManifestVersion = 10 + MaxManifestVersion = math.MaxUint8 - 230 ) type Storage struct { @@ -434,6 +435,69 @@ func (s *Storage) FindManifestsByLastAppliedRange(start, end time.Time) ([]manif return s.parseSearchResults(searchResults) } +func (s *Storage) Rollback(id string, targetVersion int) error { + return instance.db.Update(func(txn *badger.Txn) error { + key := genVersionedKey(id, targetVersion) + item, err := txn.Get(key) + if err != nil { + return fmt.Errorf("old version for manifest %s not found", id) + } + + val, _ := item.ValueCopy(nil) + newVersion, err := s.getCurrentVersion(txn, id) + if err != nil { + return err + } + + newVersion += 1 + newKey := genVersionedKey(id, newVersion) + + if err = txn.Set(newKey, val); err != nil && errors.Is(err, badger.ErrKeyNotFound) { + return fmt.Errorf("old version for manifest %s not found", id) + } else if err != nil { + return fmt.Errorf("rollback failed: %w", err) + } + + latestKey := genLatestKey(id) + return txn.Set(latestKey, newKey) + }) +} + +func (s *Storage) CleanupOldVersions(id string, keep int) error { + return instance.db.Update(func(txn *badger.Txn) error { + versions, err := s.getAllVersions(txn, id) + if err != nil { + return err + } + + if len(versions) <= keep { + return nil + } + + sort.Ints(versions) + toDelete := versions[:len(versions)-keep] + + for _, ver := range toDelete { + key := genVersionedKey(id, ver) + + if err = txn.Delete(key); err != nil { + return fmt.Errorf("error deleting old versions: %w", err) + } + + if err = s.index.Delete(string(key)); err != nil { + return fmt.Errorf("failed to delete from index: %v", err) + } + } + + latestVer := versions[len(versions)-1] + if latestVer > MaxManifestVersion { + return s.resetVersions(txn, id) + } + + return nil + }) +} + func (s *Storage) parseSearchResults(searchResults *bleve.SearchResult) ([]manifests.Manifest, error) { if searchResults == nil || searchResults.Total == 0 { return nil, nil diff --git a/internal/core/store/independ.go b/internal/core/store/independ.go index 946485b..8416fd2 100644 --- a/internal/core/store/independ.go +++ b/internal/core/store/independ.go @@ -175,6 +175,22 @@ func FindManifestsByLastAppliedRange(start, end time.Time) ([]manifests.Manifest return instance.FindManifestsByLastAppliedRange(start, end) } +func Rollback(id string, targetVersion int) error { + if !isEnabled() { + return nil + } + + return instance.Rollback(id, targetVersion) +} + +func CleanupOldVersions(id string, keep int) error { + if !isEnabled() { + return nil + } + + return instance.CleanupOldVersions(id, keep) +} + func isEnabled() bool { if !IsEnabled() { ui.Errorf("Database instance not ready") diff --git a/ui/styles.go b/ui/styles.go index e475c64..4893a94 100644 --- a/ui/styles.go +++ b/ui/styles.go @@ -18,8 +18,8 @@ var ( Foreground(lipgloss.Color("#5fd700")) errorStyle = lipgloss.NewStyle(). - Foreground(lipgloss.Color("#d70000")). - Bold(true) + Foreground(lipgloss.Color("#FF0000")). + Bold(false) warningStyle = lipgloss.NewStyle(). Foreground(lipgloss.Color("#ff8700")). From fe4d7db732c34b3c56b79e37c617f4792e80f359 Mon Sep 17 00:00:00 2001 From: Nofre Date: Sun, 18 May 2025 05:43:37 +0200 Subject: [PATCH 6/6] feat: a lof of updates, cmd, store, kins --- cmd/cli/apply.go | 2 +- cmd/cli/cleanup.go | 1 - cmd/cli/rollback.go | 1 - cmd/cli/search.go | 572 ++++++++++++++++- examples/simple/server.yaml | 2 +- examples/values/values.yaml | 2 +- .../core/manifests/kinds/values/values.go | 6 +- internal/core/manifests/loader/loader.go | 8 +- internal/core/store/db.go | 575 ++++++++---------- internal/core/store/helpers.go | 8 + internal/core/store/independ.go | 148 +---- internal/core/store/index.go | 15 +- internal/core/store/query.go | 183 ++++++ 13 files changed, 1022 insertions(+), 501 deletions(-) create mode 100644 internal/core/store/query.go diff --git a/cmd/cli/apply.go b/cmd/cli/apply.go index 7217b71..84de9d8 100644 --- a/cmd/cli/apply.go +++ b/cmd/cli/apply.go @@ -37,7 +37,7 @@ var applyCmd = &cobra.Command{ ui.Spinner(false) ui.Spinner(true, "Saving manifests...") - if err := store.SaveManifests(loadedMans...); err != nil { + if err := store.Save(loadedMans...); err != nil { ui.Error("Failed to save manifests: " + err.Error()) ui.Spinner(false) return diff --git a/cmd/cli/cleanup.go b/cmd/cli/cleanup.go index aef8f77..83050f5 100644 --- a/cmd/cli/cleanup.go +++ b/cmd/cli/cleanup.go @@ -21,7 +21,6 @@ var cleanupCmd = &cobra.Command{ RunE: func(cmd *cobra.Command, args []string) error { opts, err := parseCleanUpFlags(cmd, args) if err != nil { - ui.Errorf("Failed to parse provided values: %v", err) return err } diff --git a/cmd/cli/rollback.go b/cmd/cli/rollback.go index 0f315da..77deef2 100644 --- a/cmd/cli/rollback.go +++ b/cmd/cli/rollback.go @@ -17,7 +17,6 @@ var rollbackCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { opts, err := parseRollbackFlags(cmd, args) if err != nil { - ui.Errorf("Failed to parse provided values: %v", err) return } diff --git a/cmd/cli/search.go b/cmd/cli/search.go index e03d0f7..8ef36ae 100644 --- a/cmd/cli/search.go +++ b/cmd/cli/search.go @@ -1,40 +1,568 @@ package cli -import "github.com/spf13/cobra" +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/apiqube/cli/internal/core/manifests" + "github.com/apiqube/cli/internal/core/store" + "github.com/apiqube/cli/ui" + "github.com/spf13/cobra" + "gopkg.in/yaml.v3" +) var searchCmd = &cobra.Command{ - Use: "search", - Short: "Search for manifests using filters", + Use: "search", + Short: "Search for manifests using filters", + Long: fmt.Sprint("Search for manifests with powerful filtering options including exact/wildcard matching," + + "\ntime ranges, and output formatting"), SilenceUsage: true, SilenceErrors: true, - Run: func(cmd *cobra.Command, args []string) {}, + RunE: func(cmd *cobra.Command, args []string) error { + opts, err := parseSearchFlags(cmd, args) + if err != nil { + ui.Errorf("Failed to parse provided values: %v", err) + return err + } + + var manifests []manifests.Manifest + + if !opts.All && + !opts.flagsSet["name"] && + !opts.flagsSet["name-wildcard"] && + !opts.flagsSet["name-regex"] && + !opts.flagsSet["kind"] && + !opts.flagsSet["hash"] && + !opts.flagsSet["version"] && + !opts.flagsSet["namespace"] && + !opts.flagsSet["created-by"] && + !opts.flagsSet["used-by"] && + !opts.flagsSet["depends"] && + !opts.flagsSet["depends-all"] && + !opts.flagsSet["created-after"] && + !opts.flagsSet["created-before"] && + !opts.flagsSet["updated-after"] && + !opts.flagsSet["updated-before"] && + !opts.flagsSet["last-applied"] { + return fmt.Errorf("at least one search filter must be specified") + } + + if opts.flagsSet["all"] { + manifests, err = store.Load(store.LoadOptions{All: true}) + if err != nil { + ui.Errorf("Failed to loadmanifests: %v", err) + return nil + } + } else { + query := store.NewQuery() + + if opts.flagsSet["name"] { + query.WithExactName(opts.Name) + } else if opts.flagsSet["name-wildcard"] { + query.WithWildcardName(opts.NameWildcard) + } else if opts.flagsSet["name-regex"] { + query.WithRegexName(opts.NameRegex) + } + + if opts.flagsSet["namespace"] { + query.WithNamespace(opts.Namespace) + } + + if opts.flagsSet["kind"] { + query.WithKind(opts.Kind) + } + + if opts.flagsSet["version"] { + query.WithVersion(opts.Version) + } + + if opts.flagsSet["created-by"] { + query.WithCreatedBy(opts.CreatedBy) + } + + if opts.flagsSet["user-by"] { + query.WithUsedBy(opts.UsedBy) + } + + if opts.flagsSet["hash"] { + query.WithHashPrefix(opts.HashPrefix) + } + + if opts.flagsSet["depends"] { + query.WithDependencies(opts.DependsOn) + } else if opts.flagsSet["depends-all"] { + query.WithAllDependencies(opts.DependsOnAll) + } + + if opts.flagsSet["created-after"] { + query.WithCreatedAfter(opts.CreatedAfter) + } + + if opts.flagsSet["created-before"] { + query.WithCreatedBefore(opts.CreatedBefore) + } + + if opts.flagsSet["updated-after"] { + query.WithUpdatedAfter(opts.UpdatedAfter) + } + + if opts.flagsSet["updated-before"] { + query.WithUpdatedBefore(opts.UpdatedBefore) + } + + if opts.flagsSet["last-applied"] { + query.WithLastApplied(opts.LastApplied) + } + + manifests, err = store.Search(query) + if err != nil { + ui.Errorf("Failed to search manifests: %v", err) + return nil + } + } + + if len(manifests) == 0 { + ui.Warning("No manifests found matching the criteria") + return nil + } + + ui.Infof("Found %d manifests", len(manifests)) + + if len(opts.SortBy) > 0 { + sortManifests(manifests, opts.SortBy) + } + + ui.Spinner(true, "Prepare answer...") + + if opts.Output { + if err := outputManifests(manifests, opts); err != nil { + ui.Spinner(false) + ui.Errorf("Failed to output manifests: %v", err) + return nil + } + } else { + displayResults(manifests) + } + + ui.Spinner(false, "Complete") + + return nil + }, } func init() { - // All - searchCmd.Flags().StringP("all", "a", "", "Get all manifests") + searchCmd.Flags().BoolP("all", "a", false, "Get all manifests") - // Exact match filters - searchCmd.Flags().StringP("name", "n", "", "Search manifest by exact name match") - searchCmd.Flags().StringP("namespace", "s", "default", "Search manifests by exact namespace)") - searchCmd.Flags().StringP("kind", "k", "", "Search manifests by exact kind (e.g., Server, HttpTest, HttpLoadTest)") - searchCmd.Flags().IntP("version", "v", 1, "Search manifests by exact version number") + searchCmd.Flags().StringP("name", "n", "", "Search manifest by name (exact match)") + searchCmd.Flags().StringP("name-wildcard", "W", "", "Search manifest by wildcard pattern (e.g. '*name*')") + searchCmd.Flags().StringP("name-regex", "R", "", "Search manifest by regex pattern") - // Wildcard/partial match filters - searchCmd.Flags().StringP("wildcard", "w", "", "Search manifests using name wildcard pattern") - searchCmd.Flags().StringP("hash", "h", "", "Search manifests by hash prefix") + searchCmd.Flags().StringP("namespace", "s", "", "Search manifests by namespace") + searchCmd.Flags().StringP("kind", "k", "", "Search manifests by kind") + searchCmd.Flags().IntP("version", "v", 0, "Search manifests by version") + searchCmd.Flags().String("created-by", "", "Filter by exact creator username") + searchCmd.Flags().String("used-by", "", "Filter by exact user who applied") - // Dependency filters + searchCmd.Flags().StringP("hash", "H", "", "Search manifests by hash prefix (min 5 chars)") searchCmd.Flags().StringSliceP("depends", "d", []string{}, "Search manifests by dependencies (comma separated)") + searchCmd.Flags().StringSliceP("depends-all", "D", []string{}, "Search manifests by all dependencies (comma separated)") - // Date filters - searchCmd.Flags().String("created-after", "", "Search manifests created after date (YYYY-MM-DD)") - searchCmd.Flags().String("created-before", "", "Search manifests created before date") - searchCmd.Flags().String("updated-after", "", "Search manifests updated after date") - searchCmd.Flags().String("last-applied", "", "Search manifests by last applied date") + searchCmd.Flags().String("created-after", "", "Search manifests created after date (YYYY-MM-DD or duration like 1h30m)") + searchCmd.Flags().String("created-before", "", "Search manifests created before date/duration") + searchCmd.Flags().String("updated-after", "", "Search manifests updated after date/duration") + searchCmd.Flags().String("updated-before", "", "Search manifests updated before date/duration") + searchCmd.Flags().String("last-applied", "", "Search manifests by last applied date/duration") - // Creator filter - searchCmd.Flags().String("created-by", "", "Filter by creator username") + searchCmd.Flags().BoolP("output", "o", false, "Make output after searching") + searchCmd.Flags().String("output-path", "", "Output path for results (default: current directory)") + searchCmd.Flags().String("output-mode", "separate", "Output mode (combined|separate)") + searchCmd.Flags().String("output-format", "yaml", "File format for output (yaml|json)") + + searchCmd.Flags().StringSlice("sort", []string{}, "Sort by fields (e.g. --sort=kind,-name)") rootCmd.AddCommand(searchCmd) } + +type SearchOptions struct { + All bool + + Name string + NameWildcard string + NameRegex string + + Namespace string + Kind string + Version int + CreatedBy string + UsedBy string + + HashPrefix string + DependsOn []string + DependsOnAll []string + + CreatedAfter time.Time + CreatedBefore time.Time + UpdatedAfter time.Time + UpdatedBefore time.Time + LastApplied time.Time + IsRelativeTime bool + + Output bool + OutputPath string + OutputMode string // combined | separate + OutputFormat string // yaml | json + + SortBy []string + + flagsSet map[string]bool +} + +func parseSearchFlags(cmd *cobra.Command, _ []string) (*SearchOptions, error) { + opts := &SearchOptions{ + flagsSet: make(map[string]bool), + } + + markFlag := func(name string) bool { + if cmd.Flags().Changed(name) { + opts.flagsSet[name] = true + return true + } + return false + } + + if markFlag("all") { + opts.All, _ = cmd.Flags().GetBool("all") + } + + if markFlag("name") { + opts.Name, _ = cmd.Flags().GetString("name") + } + if markFlag("name-wildcard") { + opts.NameWildcard, _ = cmd.Flags().GetString("name-wildcard") + } + if markFlag("name-regex") { + opts.NameRegex, _ = cmd.Flags().GetString("name-regex") + } + + if opts.flagsSet["name"] && (opts.flagsSet["name-wildcard"] || opts.flagsSet["name-regex"]) { + return nil, fmt.Errorf("cannot use exact name filter with wildcard/regex filters") + } + + if markFlag("namespace") { + opts.Namespace, _ = cmd.Flags().GetString("namespace") + } + if markFlag("kind") { + opts.Kind, _ = cmd.Flags().GetString("kind") + } + if markFlag("version") { + opts.Version, _ = cmd.Flags().GetInt("version") + } + if markFlag("created-by") { + opts.CreatedBy, _ = cmd.Flags().GetString("created-by") + } + if markFlag("used-by") { + opts.UsedBy, _ = cmd.Flags().GetString("used-by") + } + + if markFlag("hash") { + opts.HashPrefix, _ = cmd.Flags().GetString("hash") + if len(opts.HashPrefix) < 5 { + return nil, fmt.Errorf("hash prefix must be at least 5 characters") + } + } + if markFlag("depends") { + opts.DependsOn, _ = cmd.Flags().GetStringSlice("depends") + } else if markFlag("depends-all") { + opts.DependsOnAll, _ = cmd.Flags().GetStringSlice("depends-all") + } + + timeFilters := map[string]*time.Time{ + "created-after": &opts.CreatedAfter, + "created-before": &opts.CreatedBefore, + "updated-after": &opts.UpdatedAfter, + "updated-before": &opts.UpdatedBefore, + "last-applied": &opts.LastApplied, + } + + for flag, target := range timeFilters { + if markFlag(flag) { + val, _ := cmd.Flags().GetString(flag) + if t, err := parseTimeOrDuration(val); err == nil { + *target = t + opts.IsRelativeTime = isDuration(val) + } else { + return nil, fmt.Errorf("invalid %s value: %w", flag, err) + } + } + } + + if markFlag("output") { + opts.Output, _ = cmd.Flags().GetBool("output") + if opts.Output { + if markFlag("output-path") { + opts.OutputPath, _ = cmd.Flags().GetString("output-path") + } + if opts.OutputPath == "" { + opts.OutputPath = "." + } + if markFlag("output-mode") { + opts.OutputMode, _ = cmd.Flags().GetString("output-mode") + if opts.OutputMode != "combined" && opts.OutputMode != "separate" { + return nil, fmt.Errorf("invalid output mode, must be 'combined' or 'separate'") + } + } + if opts.OutputMode == "" { + opts.OutputMode = "separate" + } + if markFlag("output-format") { + opts.OutputFormat, _ = cmd.Flags().GetString("output-format") + if opts.OutputFormat != "yaml" && opts.OutputFormat != "json" { + return nil, fmt.Errorf("invalid output format, must be 'yaml' or 'json'") + } + } + if opts.OutputFormat == "" { + opts.OutputFormat = "yaml" + } + } + } + + if markFlag("sort") { + opts.SortBy, _ = cmd.Flags().GetStringSlice("sort") + } + + return opts, nil +} + +func parseTimeOrDuration(val string) (time.Time, error) { + if duration, err := time.ParseDuration(val); err == nil { + return time.Now().Add(-duration), nil + } + + if t, err := time.Parse("2006-01-02", val); err == nil { + return t, nil + } + + if t, err := time.Parse(time.RFC3339, val); err == nil { + return t, nil + } + + return time.Time{}, fmt.Errorf("invalid time format") +} + +func isDuration(val string) bool { + _, err := time.ParseDuration(val) + return err == nil +} + +func outputManifests(manifests []manifests.Manifest, opts *SearchOptions) error { + if err := os.MkdirAll(opts.OutputPath, 0o755); err != nil { + return fmt.Errorf("failed to create output directory: %w", err) + } + + if opts.OutputMode == "combined" { + filename := filepath.Join(opts.OutputPath, fmt.Sprintf("manifests.%s", opts.OutputFormat)) + return writeCombinedFile(filename, manifests, opts.OutputFormat) + } else { + for _, m := range manifests { + filename := filepath.Join(opts.OutputPath, fmt.Sprintf("%s.%s", m.GetID(), opts.OutputFormat)) + if err := writeSingleFile(filename, m, opts.OutputFormat); err != nil { + return err + } + } + } + + return nil +} + +func writeCombinedFile(filename string, manifests []manifests.Manifest, format string) error { + if len(manifests) == 0 { + return fmt.Errorf("no manifests to write") + } + + file, err := os.Create(filename) + if err != nil { + return fmt.Errorf("failed to create file %s: %w", filename, err) + } + defer func() { + _ = file.Close() + }() + + switch strings.ToLower(format) { + case "yaml": + encoder := yaml.NewEncoder(file) + for i, m := range manifests { + if i > 0 { + if _, err = file.WriteString("---\n"); err != nil { + return fmt.Errorf("failed to write YAML manifest: %w", err) + } + } + if err = encoder.Encode(m); err != nil { + return fmt.Errorf("failed to encode manifest %d: %w", i+1, err) + } + } + case "json": + if _, err = file.WriteString("[\n"); err != nil { + return fmt.Errorf("failed to write JSON manifest: %w", err) + } + + encoder := json.NewEncoder(file) + encoder.SetIndent("", " ") + + for i, m := range manifests { + if i > 0 { + if _, err = file.WriteString(",\n"); err != nil { + return fmt.Errorf("failed to write YAML manifest: %w", err) + } + } + if err = encoder.Encode(m); err != nil { + return fmt.Errorf("failed to encode manifest %d: %w", i+1, err) + } + } + if _, err = file.WriteString("\n]"); err != nil { + return fmt.Errorf("failed to write JSON manifest: %w", err) + } + default: + return fmt.Errorf("unsupported format: %s", format) + } + + ui.Successf("Successfully wrote %d manifests to %s", len(manifests), filename) + return nil +} + +func writeSingleFile(filename string, manifest manifests.Manifest, format string) error { + file, err := os.Create(filename) + if err != nil { + return fmt.Errorf("failed to create file %s: %w", filename, err) + } + defer func() { + _ = file.Close() + }() + + switch strings.ToLower(format) { + case "yaml": + encoder := yaml.NewEncoder(file) + if err = encoder.Encode(manifest); err != nil { + return fmt.Errorf("failed to encode manifest: %w", err) + } + case "json": + encoder := json.NewEncoder(file) + encoder.SetIndent("", " ") + if err = encoder.Encode(manifest); err != nil { + return fmt.Errorf("failed to encode manifest: %w", err) + } + default: + return fmt.Errorf("unsupported format: %s", format) + } + + ui.Successf("Successfully wrote manifest %s to %s", manifest.GetID(), filename) + return nil +} + +func sortManifests(manifests []manifests.Manifest, fields []string) { + sort.Slice(manifests, func(i, j int) bool { + for _, field := range fields { + desc := false + if strings.HasPrefix(field, "-") { + desc = true + field = field[1:] + } + + switch field { + case "id": + if manifests[i].GetID() != manifests[j].GetID() { + if desc { + return manifests[i].GetID() > manifests[j].GetID() + } + return manifests[i].GetID() < manifests[j].GetID() + } + case "name": + if manifests[i].GetName() != manifests[j].GetName() { + if desc { + return manifests[i].GetName() > manifests[j].GetName() + } + return manifests[i].GetName() < manifests[j].GetName() + } + case "kind": + if manifests[i].GetKind() != manifests[j].GetKind() { + if desc { + return manifests[i].GetKind() > manifests[j].GetKind() + } + return manifests[i].GetKind() < manifests[j].GetKind() + } + case "namespace": + if manifests[i].GetNamespace() != manifests[j].GetNamespace() { + if desc { + return manifests[i].GetNamespace() > manifests[j].GetNamespace() + } + return manifests[i].GetNamespace() < manifests[j].GetNamespace() + } + case "version": + if desc { + return manifests[i].GetMeta().GetVersion() > manifests[j].GetMeta().GetVersion() + } + return manifests[i].GetMeta().GetVersion() < manifests[j].GetMeta().GetVersion() + case "created": + if desc { + return manifests[i].GetMeta().GetCreatedAt().After(manifests[j].GetMeta().GetCreatedAt()) + } + return manifests[i].GetMeta().GetCreatedAt().Before(manifests[j].GetMeta().GetCreatedAt()) + case "updated": + if desc { + return manifests[i].GetMeta().GetUpdatedAt().After(manifests[j].GetMeta().GetUpdatedAt()) + } + return manifests[i].GetMeta().GetUpdatedAt().Before(manifests[j].GetMeta().GetUpdatedAt()) + case "last": + if desc { + return manifests[i].GetMeta().GetLastApplied().After(manifests[j].GetMeta().GetLastApplied()) + } + return manifests[i].GetMeta().GetLastApplied().Before(manifests[j].GetMeta().GetLastApplied()) + } + } + return false + }) +} + +func displayResults(manifests []manifests.Manifest) { + headers := []string{ + "#", + "Hash", + "Kind", + "Name", + "Namespace", + "Version", + "Created", + "Updated", + "Last Updated", + } + + var rows [][]string + for i, m := range manifests { + meta := m.GetMeta() + row := []string{ + fmt.Sprint(i + 1), + shortHash(meta.GetHash()), + m.GetKind(), + m.GetName(), + m.GetNamespace(), + fmt.Sprint(meta.GetVersion()), + meta.GetCreatedAt().Format(time.RFC3339), + meta.GetUpdatedAt().Format(time.RFC3339), + meta.GetLastApplied().Format(time.RFC3339), + } + rows = append(rows, row) + } + + ui.Table(headers, rows) +} + +func shortHash(fullHash string) string { + if len(fullHash) > 8 { + return fullHash[:8] + } + return fullHash +} diff --git a/examples/simple/server.yaml b/examples/simple/server.yaml index 9084ef8..0f60e8d 100644 --- a/examples/simple/server.yaml +++ b/examples/simple/server.yaml @@ -6,6 +6,6 @@ metadata: name: simple-server spec: - baseUrl: "http://localhost:8080" + baseUrl: "http://localhost:8081" headers: Content-Type: application/json \ No newline at end of file diff --git a/examples/values/values.yaml b/examples/values/values.yaml index 9b839c2..b8172c5 100644 --- a/examples/values/values.yaml +++ b/examples/values/values.yaml @@ -6,7 +6,7 @@ metadata: spec: users: - username: ["Max", "Carl", "John", "Alex"] + username: ["Max", "Carl", "John", "Alex", "Uli"] email: - "email_1@gmail.com" - "email_2@mail.com" diff --git a/internal/core/manifests/kinds/values/values.go b/internal/core/manifests/kinds/values/values.go index b9c281b..4b70f11 100644 --- a/internal/core/manifests/kinds/values/values.go +++ b/internal/core/manifests/kinds/values/values.go @@ -18,16 +18,12 @@ type Values struct { kinds.BaseManifest `yaml:",inline" json:",inline"` Spec struct { - Content `yaml:",inline" json:",inline"` + Data map[string]any `yaml:",inline" json:",inline"` } `yaml:"spec" valid:"required"` Meta *kinds.Meta `yaml:"-" json:"meta"` } -type Content struct { - Values map[string]any `yaml:",inline" json:",inline"` -} - func (v *Values) GetID() string { return kinds.FormManifestID(v.Namespace, v.Kind, v.Name) } diff --git a/internal/core/manifests/loader/loader.go b/internal/core/manifests/loader/loader.go index 93588a1..69c078f 100644 --- a/internal/core/manifests/loader/loader.go +++ b/internal/core/manifests/loader/loader.go @@ -90,12 +90,18 @@ func processFile(filePath string, manifestsSet map[string]struct{}) (new []manif return nil, nil, fmt.Errorf("failed to calculate hash for manifest %s: %w", manifestID, err) } + var loadedManifests []manifests.Manifest var existingManifest manifests.Manifest - existingManifest, err = store.FindManifestByHash(manifestHash) + + loadedManifests, err = store.Load(store.LoadOptions{Hash: manifestHash}) if err != nil && !isNotFoundError(err) { return nil, nil, fmt.Errorf("failed to check manifest existence: %w", err) } + if len(loadedManifests) > 0 { + existingManifest = loadedManifests[0] + } + if existingManifest != nil { if _, exists := manifestsSet[existingManifest.GetID()]; !exists { manifestsSet[existingManifest.GetID()] = struct{}{} diff --git a/internal/core/store/db.go b/internal/core/store/db.go index 33ee985..e36316d 100644 --- a/internal/core/store/db.go +++ b/internal/core/store/db.go @@ -13,11 +13,10 @@ import ( "time" "github.com/apiqube/cli/internal/core/manifests/index" - "github.com/apiqube/cli/internal/core/manifests/parsing" - bleceQuery "github.com/blevesearch/bleve/v2/search/query" "github.com/adrg/xdg" "github.com/apiqube/cli/internal/core/manifests" + "github.com/apiqube/cli/internal/core/manifests/parsing" "github.com/blevesearch/bleve/v2" "github.com/dgraph-io/badger/v4" ) @@ -30,6 +29,16 @@ const ( MaxManifestVersion = math.MaxUint8 - 230 ) +type LoadOptions struct { + IDs []string + Versions map[string]int // id -> version + Hash string + Latest bool + All bool + SortBy []string + Limit int +} + type Storage struct { db *badger.DB index bleve.Index @@ -69,7 +78,7 @@ func NewStorage() (*Storage, error) { }, nil } -func (s *Storage) SaveManifests(mans ...manifests.Manifest) error { +func (s *Storage) Save(mans ...manifests.Manifest) error { return instance.db.Update(func(txn *badger.Txn) error { for _, m := range mans { currentVer, err := s.getCurrentVersion(txn, m.GetID()) @@ -95,6 +104,7 @@ func (s *Storage) SaveManifests(mans ...manifests.Manifest) error { if err = txn.Set(versionedKey, data); err != nil { return err } + if err = txn.Set(latestKey, versionedKey); err != nil { return err } @@ -113,326 +123,46 @@ func (s *Storage) SaveManifests(mans ...manifests.Manifest) error { }) } -func (s *Storage) LoadManifests(ids ...string) ([]manifests.Manifest, error) { - if len(ids) == 0 { - return nil, fmt.Errorf("empty IDs list") - } - - var ( - results []manifests.Manifest - errs error - ) - - err := instance.db.View(func(txn *badger.Txn) error { - for _, id := range ids { - item, err := txn.Get(genLatestKey(id)) - if err != nil { - if errors.Is(err, badger.ErrKeyNotFound) { - errs = errors.Join(errs, fmt.Errorf("manifest %q not found", id)) - continue - } - errs = errors.Join(errs, fmt.Errorf("failed to get latest pointer for %q: %w", id, err)) - continue - } - - var latestKey []byte - if err = item.Value(func(val []byte) error { - latestKey = append([]byte{}, val...) - return nil - }); err != nil { - errs = errors.Join(errs, fmt.Errorf("failed to read latest key for %q: %w", id, err)) - continue - } - - item, err = txn.Get(latestKey) - if err != nil { - errs = errors.Join(errs, fmt.Errorf("failed to get manifest data for %q: %w", id, err)) - continue - } - - var manifest manifests.Manifest - if err = item.Value(func(data []byte) error { - manifest, err = parsing.ParseManifestAsJSON(data) - return err - }); err != nil { - errs = errors.Join(errs, fmt.Errorf("parsing failed for %q: %w", id, err)) - continue - } - - results = append(results, manifest) - } - - if len(results) == 0 && errs != nil { - return fmt.Errorf("no manifests loaded: %w", errs) - } - return nil - }) - if err != nil { - return results, fmt.Errorf("transaction failed: %w", errors.Join(err, errs)) - } - - return results, errs -} - -func (s *Storage) LoadManifest(id string) (manifests.Manifest, error) { - var manifest manifests.Manifest - - err := instance.db.View(func(txn *badger.Txn) error { - item, err := txn.Get(genLatestKey(id)) - if err != nil { - return fmt.Errorf("failed to get latest version pointer: %w", err) - } +func (s *Storage) Load(opts LoadOptions) ([]manifests.Manifest, error) { + var results []manifests.Manifest - var versionedKey []byte - if err := item.Value(func(val []byte) error { - versionedKey = make([]byte, len(val)) - copy(versionedKey, val) - return nil - }); err != nil { - return fmt.Errorf("failed to read version key: %w", err) - } - - item, err = txn.Get(versionedKey) + if opts.Hash != "" { + m, err := s.loadByHash(opts.Hash) if err != nil { - return fmt.Errorf("failed to get manifest data: %w", err) + return nil, err } - - if err = item.Value(func(data []byte) error { - var parseErr error - manifest, parseErr = parsing.ParseManifestAsJSON(data) - return parseErr - }); err != nil { - return fmt.Errorf("parsing failed: %w", err) - } - - return nil - }) - if err != nil { - return nil, fmt.Errorf("failed to load manifest %q: %w", id, err) - } - - return manifest, nil -} - -func (s *Storage) FindManifestsByKind(kind string) ([]manifests.Manifest, error) { - query := bleve.NewTermQuery(kind) - query.SetField(index.Kind) - - searchRequest := bleve.NewSearchRequest(query) - - searchResults, err := s.index.Search(searchRequest) - if err != nil { - return nil, fmt.Errorf("error searching manifests: %v", err) - } - - return s.parseSearchResults(searchResults) -} - -func (s *Storage) FindManifestsByVersion(lowVersion, heightVersion uint8) ([]manifests.Manifest, error) { - low, height := float64(lowVersion), float64(heightVersion) - - query := bleve.NewNumericRangeQuery(&low, &height) - query.SetField(index.Version) - - searchRequest := bleve.NewSearchRequest(query) - - searchResults, err := s.index.Search(searchRequest) - if err != nil { - return nil, fmt.Errorf("error searching manifests: %v", err) - } - - return s.parseSearchResults(searchResults) -} - -func (s *Storage) FindManifestByName(name string) (manifests.Manifest, error) { - query := bleve.NewMatchPhraseQuery(name) - query.SetField(index.Name) - - searchRequest := bleve.NewSearchRequest(query) - searchRequest.Size = 1 - - searchResults, err := s.index.Search(searchRequest) - if err != nil { - return nil, fmt.Errorf("search failed: %w", err) + return []manifests.Manifest{m}, nil } - results, err := s.parseSearchResults(searchResults) - if err != nil { - return nil, err - } - - if len(results) == 0 { - return nil, fmt.Errorf("no matching manifest found") - } else { - return results[0], nil - } -} - -func (s *Storage) FindManifestsByNameWildcard(namePattern string) ([]manifests.Manifest, error) { - query := bleve.NewWildcardQuery(fmt.Sprintf("*%s*", namePattern)) - query.SetField(index.Name) - - searchRequest := bleve.NewSearchRequest(query) - - searchResults, err := s.index.Search(searchRequest) - if err != nil { - return nil, fmt.Errorf("search failed: %w", err) - } - - return s.parseSearchResults(searchResults) -} - -func (s *Storage) FindManifestsByNamespace(namespace string) ([]manifests.Manifest, error) { - query := bleve.NewTermQuery(namespace) - query.SetField(index.Namespace) - - searchRequest := bleve.NewSearchRequest(query) - - searchResults, err := s.index.Search(searchRequest) - if err != nil { - return nil, fmt.Errorf("search failed: %w", err) - } - - return s.parseSearchResults(searchResults) -} - -func (s *Storage) FindManifestByDependencies(dependencies []string, requireAll bool) ([]manifests.Manifest, error) { - var query bleceQuery.Query - - if requireAll { - q := bleve.NewConjunctionQuery() - for _, dep := range dependencies { - termQuery := bleve.NewTermQuery(dep) - termQuery.SetField(index.DependsOn) - q.AddQuery(termQuery) - } - query = q - } else { - q := bleve.NewDisjunctionQuery() - for _, dep := range dependencies { - termQuery := bleve.NewTermQuery(dep) - termQuery.SetField(index.DependsOn) - q.AddQuery(termQuery) + if len(opts.Versions) > 0 { + for id, version := range opts.Versions { + m, err := s.loadVersion(id, version) + if err != nil { + return nil, fmt.Errorf("failed to load %s@v%d: %w", id, version, err) + } + results = append(results, m) } - query = q + return results, nil } - searchRequest := bleve.NewSearchRequest(query) - - searchResults, err := s.index.Search(searchRequest) - if err != nil { - return nil, fmt.Errorf("search failed: %w", err) - } - - return s.parseSearchResults(searchResults) -} - -func (s *Storage) FindManifestByHash(hash string) (manifests.Manifest, error) { - query := bleve.NewTermQuery(hash) - query.SetField(index.MetaHash) - - searchRequest := bleve.NewSearchRequest(query) - searchRequest.Size = 1 - - searchResults, err := s.index.Search(searchRequest) - if err != nil { - return nil, fmt.Errorf("search failed: %w", err) - } - - results, err := s.parseSearchResults(searchResults) - if err != nil { - return nil, err - } - - if len(results) == 0 { - return nil, fmt.Errorf("no matching manifest found") - } else { - return results[0], nil - } -} - -func (s *Storage) FindManifestsByCreatedAtRange(start, end time.Time) ([]manifests.Manifest, error) { - query := bleve.NewDateRangeQuery(start, end) - query.SetField(index.MetaCreatedAt) - - searchRequest := bleve.NewSearchRequest(query) - - searchResults, err := s.index.Search(searchRequest) - if err != nil { - return nil, fmt.Errorf("error searching manifests: %v", err) - } - - return s.parseSearchResults(searchResults) -} - -func (s *Storage) FindManifestsByCreatedBy(createdBy string) ([]manifests.Manifest, error) { - query := bleve.NewTermQuery(createdBy) - query.SetField(index.MetaCreatedBy) - - searchRequest := bleve.NewSearchRequest(query) - - searchResults, err := s.index.Search(searchRequest) - if err != nil { - return nil, fmt.Errorf("search failed: %w", err) - } - - return s.parseSearchResults(searchResults) -} - -func (s *Storage) FindManifestsByUpdatedAtRange(start, end time.Time) ([]manifests.Manifest, error) { - query := bleve.NewDateRangeQuery(start, end) - query.SetField(index.MetaUpdatedAt) - - searchRequest := bleve.NewSearchRequest(query) - - searchResults, err := s.index.Search(searchRequest) - if err != nil { - return nil, fmt.Errorf("error searching manifests: %v", err) - } - - return s.parseSearchResults(searchResults) -} - -func (s *Storage) FindManifestsByUpdatedBy(updatedBy string) ([]manifests.Manifest, error) { - query := bleve.NewTermQuery(updatedBy) - query.SetField(index.MetaUpdatedBy) - - searchRequest := bleve.NewSearchRequest(query) - - searchResults, err := s.index.Search(searchRequest) - if err != nil { - return nil, fmt.Errorf("search failed: %w", err) + if len(opts.IDs) > 0 { + return s.loadBulk(opts) } - return s.parseSearchResults(searchResults) -} - -func (s *Storage) FindManifestsByUsedBy(usedBy string) ([]manifests.Manifest, error) { - query := bleve.NewTermQuery(usedBy) - query.SetField(index.MetaUsedBy) - - searchRequest := bleve.NewSearchRequest(query) - - searchResults, err := s.index.Search(searchRequest) - if err != nil { - return nil, fmt.Errorf("search failed: %w", err) + if opts.All || opts.Latest { + return s.loadLatest() } - return s.parseSearchResults(searchResults) + return nil, fmt.Errorf("no valid load options specified") } -func (s *Storage) FindManifestsByLastAppliedRange(start, end time.Time) ([]manifests.Manifest, error) { - query := bleve.NewDateRangeQuery(start, end) - query.SetField(index.MetaLastApplied) - - searchRequest := bleve.NewSearchRequest(query) - - searchResults, err := s.index.Search(searchRequest) +func (s *Storage) Search(query Query) ([]manifests.Manifest, error) { + searchResult, err := query.Execute(s.index) if err != nil { - return nil, fmt.Errorf("error searching manifests: %v", err) + return nil, fmt.Errorf("failed to search manifests: %w", err) } - return s.parseSearchResults(searchResults) + return s.parseSearchResults(searchResult) } func (s *Storage) Rollback(id string, targetVersion int) error { @@ -498,55 +228,230 @@ func (s *Storage) CleanupOldVersions(id string, keep int) error { }) } -func (s *Storage) parseSearchResults(searchResults *bleve.SearchResult) ([]manifests.Manifest, error) { - if searchResults == nil || searchResults.Total == 0 { - return nil, nil +func (s *Storage) loadByHash(hash string) (manifests.Manifest, error) { + var manifest manifests.Manifest + + hashQuery := bleve.NewTermQuery(hash) + hashQuery.SetField(index.MetaHash) + + searchResult, err := s.index.Search(bleve.NewSearchRequest(hashQuery)) + if err != nil { + return nil, fmt.Errorf("bleve search failed: %w", err) } - var ( - manifestsList = make([]manifests.Manifest, 0, len(searchResults.Hits)) - errorsList []error - ) + if searchResult.Total == 0 { + return nil, fmt.Errorf("manifest with hash %q not found", hash) + } - for _, hit := range searchResults.Hits { - manifest, err := s.loadManifestByID(hit.ID) + hit := searchResult.Hits[0] + + err = s.db.View(func(txn *badger.Txn) error { + var latestItem *badger.Item + + latestKey := genLatestKey(extractBaseID(hit.ID)) + latestItem, err = txn.Get(latestKey) if err != nil { - errorsList = append(errorsList, fmt.Errorf("id %q: %w", hit.ID, err)) - continue + return fmt.Errorf("failed to get latest version: %w", err) + } + var versionedKey []byte + versionedKey, err = latestItem.ValueCopy(nil) + if err != nil { + return fmt.Errorf("failed to get versioned key: %w", err) } - if manifest != nil { - manifestsList = append(manifestsList, manifest) + var manifestItem *badger.Item + manifestItem, err = txn.Get(versionedKey) + if err != nil { + return fmt.Errorf("failed to get manifest data: %w", err) } + + return manifestItem.Value(func(data []byte) error { + var parseErr error + manifest, parseErr = parsing.ParseManifest(parsing.JSONMethod, data) + return parseErr + }) + }) + if err != nil { + return nil, fmt.Errorf("storage error: %w", err) } - return manifestsList, errors.Join(errorsList...) + if manifest.GetMeta().GetHash() != hash { + return nil, fmt.Errorf("hash mismatch after load") + } + + return manifest, nil } -func (s *Storage) loadManifestByID(id string) (manifests.Manifest, error) { - var manifest manifests.Manifest +func (s *Storage) loadVersion(id string, version int) (manifests.Manifest, error) { + var m manifests.Manifest + versionedKey := genVersionedKey(id, version) - err := instance.db.View(func(txn *badger.Txn) error { - item, err := txn.Get([]byte(id)) + err := s.db.View(func(txn *badger.Txn) error { + item, err := txn.Get(versionedKey) if err != nil { return err } - return item.Value(func(data []byte) error { - var parseErr error - manifest, parseErr = parsing.ParseManifest(parsing.JSONMethod, data) - return parseErr + m, err = parsing.ParseManifestAsJSON(data) + return err }) }) - switch { - case errors.Is(err, badger.ErrKeyNotFound): - return nil, fmt.Errorf("not found") - case err != nil: - return nil, fmt.Errorf("storage error: %w", err) - default: - return manifest, nil + return m, err +} + +func (s *Storage) loadLatest() ([]manifests.Manifest, error) { + var results []manifests.Manifest + + err := s.db.View(func(txn *badger.Txn) error { + opts := badger.DefaultIteratorOptions + opts.Prefix = []byte(manifestsLatestKey) + it := txn.NewIterator(opts) + defer it.Close() + + for it.Rewind(); it.Valid(); it.Next() { + item := it.Item() + + versionedKey, err := item.ValueCopy(nil) + if err != nil { + return fmt.Errorf("failed to get versioned key for %s: %w", item.Key(), err) + } + + versionedItem, err := txn.Get(versionedKey) + if err != nil { + return fmt.Errorf("failed to get manifest at %s: %w", versionedKey, err) + } + + var m manifests.Manifest + err = versionedItem.Value(func(data []byte) error { + m, err = parsing.ParseManifestAsJSON(data) + if err != nil { + return fmt.Errorf("failed to parse manifest %s: %w", versionedKey, err) + } + results = append(results, m) + return nil + }) + if err != nil { + return err + } + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("failed to load results: %w", err) + } + + return results, nil +} + +func (s *Storage) loadBulk(opts LoadOptions) ([]manifests.Manifest, error) { + var ( + results []manifests.Manifest + errs error + ) + + err := instance.db.View(func(txn *badger.Txn) error { + for _, id := range opts.IDs { + item, err := txn.Get(genLatestKey(id)) + if err != nil { + if errors.Is(err, badger.ErrKeyNotFound) { + errs = errors.Join(errs, fmt.Errorf("manifest %q not found", id)) + continue + } + errs = errors.Join(errs, fmt.Errorf("failed to get latest pointer for %q: %w", id, err)) + continue + } + + var latestKey []byte + if err = item.Value(func(val []byte) error { + latestKey = append([]byte{}, val...) + return nil + }); err != nil { + errs = errors.Join(errs, fmt.Errorf("failed to read latest key for %q: %w", id, err)) + continue + } + + item, err = txn.Get(latestKey) + if err != nil { + errs = errors.Join(errs, fmt.Errorf("failed to get manifest data for %q: %w", id, err)) + continue + } + + var manifest manifests.Manifest + if err = item.Value(func(data []byte) error { + manifest, err = parsing.ParseManifestAsJSON(data) + return err + }); err != nil { + errs = errors.Join(errs, fmt.Errorf("parsing failed for %q: %w", id, err)) + continue + } + + results = append(results, manifest) + } + + if len(results) == 0 && errs != nil { + return fmt.Errorf("no manifests loaded: %w", errs) + } + return nil + }) + if err != nil { + return results, fmt.Errorf("transaction failed: %w", errors.Join(err, errs)) + } + + return results, errs +} + +func (s *Storage) parseSearchResults(searchResults *bleve.SearchResult) ([]manifests.Manifest, error) { + if searchResults == nil || searchResults.Total == 0 { + return nil, nil } + + var ( + manifestsList = make([]manifests.Manifest, 0, len(searchResults.Hits)) + errorsList []error + ) + + err := s.db.View(func(txn *badger.Txn) error { + for _, hit := range searchResults.Hits { + latestKey := genLatestKey(extractBaseID(hit.ID)) + item, err := txn.Get(latestKey) + if err != nil { + errorsList = append(errorsList, fmt.Errorf("latest key for %q: %w", hit.ID, err)) + continue + } + + versionedKey, err := item.ValueCopy(nil) + if err != nil { + errorsList = append(errorsList, fmt.Errorf("value copy for %q: %w", hit.ID, err)) + continue + } + + manifestItem, err := txn.Get(versionedKey) + if err != nil { + errorsList = append(errorsList, fmt.Errorf("manifest %q: %w", string(versionedKey), err)) + continue + } + + var m manifests.Manifest + err = manifestItem.Value(func(data []byte) error { + m, err = parsing.ParseManifest(parsing.JSONMethod, data) + if err != nil { + return err + } + manifestsList = append(manifestsList, m) + return nil + }) + if err != nil { + errorsList = append(errorsList, fmt.Errorf("parse manifest %q: %w", string(versionedKey), err)) + } + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("transaction failed: %w", err) + } + + return manifestsList, errors.Join(errorsList...) } func (s *Storage) getCurrentVersion(txn *badger.Txn, id string) (int, error) { diff --git a/internal/core/store/helpers.go b/internal/core/store/helpers.go index a0bb553..df172e1 100644 --- a/internal/core/store/helpers.go +++ b/internal/core/store/helpers.go @@ -2,6 +2,7 @@ package store import ( "fmt" + "strings" ) const ( @@ -15,3 +16,10 @@ func genLatestKey(id string) []byte { func genVersionedKey(id string, version int) []byte { return []byte(fmt.Sprintf("%s@v%d", id, version)) } + +func extractBaseID(versionedKey string) string { + if at := strings.LastIndex(versionedKey, "@"); at != -1 { + return versionedKey[:at] + } + return versionedKey +} diff --git a/internal/core/store/independ.go b/internal/core/store/independ.go index 8416fd2..d941fec 100644 --- a/internal/core/store/independ.go +++ b/internal/core/store/independ.go @@ -1,13 +1,15 @@ package store import ( + "errors" "sync" - "time" "github.com/apiqube/cli/internal/core/manifests" "github.com/apiqube/cli/ui" ) +var errStoreNotInitialized = errors.New("store not initialized") + var ( instance *Storage once sync.Once @@ -47,154 +49,42 @@ func IsEnabled() bool { return instance != nil && enabled } -func SaveManifests(mans ...manifests.Manifest) error { - if !isEnabled() { - return nil - } - - return instance.SaveManifests(mans...) -} - -func LoadManifests(ids ...string) ([]manifests.Manifest, error) { - if !isEnabled() { - return nil, nil - } - - return instance.LoadManifests() -} - -func LoadManifest(id string) (manifests.Manifest, error) { - if !isEnabled() { - return nil, nil - } - - return instance.LoadManifest(id) -} - -func FindManifestsByKind(kind string) ([]manifests.Manifest, error) { - if !isEnabled() { - return nil, nil - } - - return instance.FindManifestsByKind(kind) -} - -func FindManifestsByVersion(lowVersion, heightVersion uint8) ([]manifests.Manifest, error) { - if !isEnabled() { - return nil, nil - } - - return instance.FindManifestsByVersion(lowVersion, heightVersion) -} - -func FindManifestByName(name string) (manifests.Manifest, error) { - if !isEnabled() { - return nil, nil - } - - return instance.FindManifestByName(name) -} - -func FindManifestsByNameWildcard(namePattern string) ([]manifests.Manifest, error) { - if !isEnabled() { - return nil, nil - } - - return instance.FindManifestsByNameWildcard(namePattern) -} - -func FindManifestsByNamespace(namespace string) ([]manifests.Manifest, error) { - if !isEnabled() { - return nil, nil - } - - return instance.FindManifestsByNamespace(namespace) -} - -func FindManifestByDependencies(dependencies []string, requireAll bool) ([]manifests.Manifest, error) { - if !isEnabled() { - return nil, nil - } - - return instance.FindManifestByDependencies(dependencies, requireAll) -} - -func FindManifestByHash(hash string) (manifests.Manifest, error) { - if !isEnabled() { - return nil, nil - } - - return instance.FindManifestByHash(hash) -} - -func FindManifestsByCreatedAtRange(start, end time.Time) ([]manifests.Manifest, error) { - if !isEnabled() { - return nil, nil - } - - return instance.FindManifestsByCreatedAtRange(start, end) -} - -func FindManifestsByCreatedBy(createdBy string) ([]manifests.Manifest, error) { - if !isEnabled() { - return nil, nil - } - - return instance.FindManifestsByCreatedBy(createdBy) -} - -func FindManifestsByUpdatedAtRange(start, end time.Time) ([]manifests.Manifest, error) { - if !isEnabled() { - return nil, nil - } - - return instance.FindManifestsByUpdatedAtRange(start, end) -} - -func FindManifestsByUpdatedBy(updatedBy string) ([]manifests.Manifest, error) { - if !isEnabled() { - return nil, nil +func Save(mans ...manifests.Manifest) error { + if !IsEnabled() { + return errStoreNotInitialized } - return instance.FindManifestsByUpdatedBy(updatedBy) + return instance.Save(mans...) } -func FindManifestsByUsedBy(usedBy string) ([]manifests.Manifest, error) { - if !isEnabled() { - return nil, nil +func Load(opts LoadOptions) ([]manifests.Manifest, error) { + if !IsEnabled() { + return nil, errStoreNotInitialized } - return instance.FindManifestsByUsedBy(usedBy) + return instance.Load(opts) } -func FindManifestsByLastAppliedRange(start, end time.Time) ([]manifests.Manifest, error) { - if !isEnabled() { - return nil, nil +func Search(query Query) ([]manifests.Manifest, error) { + if !IsEnabled() { + return nil, errStoreNotInitialized } - return instance.FindManifestsByLastAppliedRange(start, end) + return instance.Search(query) } func Rollback(id string, targetVersion int) error { - if !isEnabled() { - return nil + if !IsEnabled() { + return errStoreNotInitialized } return instance.Rollback(id, targetVersion) } func CleanupOldVersions(id string, keep int) error { - if !isEnabled() { - return nil + if !IsEnabled() { + return errStoreNotInitialized } return instance.CleanupOldVersions(id, keep) } - -func isEnabled() bool { - if !IsEnabled() { - ui.Errorf("Database instance not ready") - return false - } - return true -} diff --git a/internal/core/store/index.go b/internal/core/store/index.go index 121eb2d..7792215 100644 --- a/internal/core/store/index.go +++ b/internal/core/store/index.go @@ -22,7 +22,6 @@ func buildBleveMapping() *mapping.IndexMappingImpl { exactMatchFields := []string{ index.Kind, - index.Namespace, index.DependsOn, index.MetaCreatedBy, index.MetaUpdatedBy, @@ -36,15 +35,23 @@ func buildBleveMapping() *mapping.IndexMappingImpl { } nameMapping := bleve.NewTextFieldMapping() - nameMapping.Analyzer = "standard" + nameMapping.Analyzer = "keyword" manifestMapping.AddFieldMappingsAt(index.Name, nameMapping) + namespaceMapping := bleve.NewTextFieldMapping() + namespaceMapping.Analyzer = "keyword" + manifestMapping.AddFieldMappingsAt(index.Namespace, namespaceMapping) + hashMapping := bleve.NewTextFieldMapping() hashMapping.Analyzer = "keyword" hashMapping.Store = true manifestMapping.AddFieldMappingsAt(index.MetaHash, hashMapping) - dateTimeMapping := bleve.NewDateTimeFieldMapping() + dateTimeFieldMapping := bleve.NewTextFieldMapping() + dateTimeFieldMapping.Analyzer = "keyword" + dateTimeFieldMapping.Store = true + dateTimeFieldMapping.IncludeTermVectors = false + dateTimeFieldMapping.IncludeInAll = false dateFields := []string{ index.MetaCreatedAt, @@ -53,7 +60,7 @@ func buildBleveMapping() *mapping.IndexMappingImpl { } for _, field := range dateFields { - manifestMapping.AddFieldMappingsAt(field, dateTimeMapping) + manifestMapping.AddFieldMappingsAt(field, dateTimeFieldMapping) } indexMapping.DefaultMapping = manifestMapping diff --git a/internal/core/store/query.go b/internal/core/store/query.go new file mode 100644 index 0000000..84113c3 --- /dev/null +++ b/internal/core/store/query.go @@ -0,0 +1,183 @@ +package store + +import ( + "time" + + "github.com/apiqube/cli/internal/core/manifests/index" + "github.com/blevesearch/bleve/v2" + "github.com/blevesearch/bleve/v2/search/query" +) + +type Query interface { + Execute(index bleve.Index) (*bleve.SearchResult, error) + WithAll() Query + WithExactName(name string) Query + WithWildcardName(pattern string) Query + WithRegexName(regex string) Query + WithKind(kind string) Query + WithNamespace(namespace string) Query + WithVersion(version int) Query + WithCreatedBy(by string) Query + WithUsedBy(by string) Query + WithHashPrefix(prefix string) Query + WithDependencies(deps []string) Query + WithAllDependencies(deps []string) Query + WithCreatedAfter(t time.Time) Query + WithCreatedBefore(t time.Time) Query + WithUpdatedAfter(t time.Time) Query + WithUpdatedBefore(t time.Time) Query + WithLastApplied(t time.Time) Query +} + +type manifestQuery struct { + bleveQuery query.Query + limit int +} + +func NewQuery() Query { + return &manifestQuery{ + bleveQuery: bleve.NewMatchAllQuery(), + } +} + +func (q *manifestQuery) Execute(index bleve.Index) (*bleve.SearchResult, error) { + searchRequest := bleve.NewSearchRequest(q.bleveQuery) + if q.limit > 0 { + searchRequest.Size = q.limit + } + + return index.Search(searchRequest) +} + +func (q *manifestQuery) WithAll() Query { + q.bleveQuery = bleve.NewMatchAllQuery() + return q +} + +func (q *manifestQuery) WithExactName(name string) Query { + termQuery := bleve.NewTermQuery(name) + termQuery.SetField(index.Name) + q.addQuery(termQuery) + return q +} + +func (q *manifestQuery) WithWildcardName(pattern string) Query { + wildcardQuery := bleve.NewWildcardQuery(pattern) + wildcardQuery.SetField(index.Name) + q.addQuery(wildcardQuery) + return q +} + +func (q *manifestQuery) WithRegexName(regex string) Query { + regexpQuery := bleve.NewRegexpQuery(regex) + regexpQuery.SetField(index.Name) + q.addQuery(regexpQuery) + return q +} + +func (q *manifestQuery) WithKind(kind string) Query { + termQuery := bleve.NewTermQuery(kind) + termQuery.SetField(index.Kind) + q.addQuery(termQuery) + return q +} + +func (q *manifestQuery) WithNamespace(namespace string) Query { + termQuery := bleve.NewTermQuery(namespace) + termQuery.SetField(index.Namespace) + q.addQuery(termQuery) + return q +} + +func (q *manifestQuery) WithVersion(version int) Query { + val := float64(version) + numericQuery := bleve.NewNumericRangeQuery(&val, nil) + numericQuery.SetField(index.MetaVersion) + return q +} + +func (q *manifestQuery) WithCreatedBy(by string) Query { + termQuery := bleve.NewTermQuery(by) + termQuery.SetField(index.MetaCreatedBy) + q.addQuery(termQuery) + return q +} + +func (q *manifestQuery) WithUsedBy(by string) Query { + termQuery := bleve.NewTermQuery(by) + termQuery.SetField(index.MetaUsedBy) + q.addQuery(termQuery) + return q +} + +func (q *manifestQuery) WithHashPrefix(prefix string) Query { + prefixQuery := bleve.NewPrefixQuery(prefix) + prefixQuery.SetField(index.MetaHash) + q.addQuery(prefixQuery) + return q +} + +func (q *manifestQuery) WithDependencies(deps []string) Query { + disjunctionQuery := bleve.NewDisjunctionQuery() + for _, dep := range deps { + termQuery := bleve.NewTermQuery(dep) + termQuery.SetField(index.DependsOn) + disjunctionQuery.AddQuery(termQuery) + } + q.addQuery(disjunctionQuery) + return q +} + +func (q *manifestQuery) WithAllDependencies(deps []string) Query { + conjunctionQuery := bleve.NewConjunctionQuery() + for _, dep := range deps { + termQuery := bleve.NewTermQuery(dep) + termQuery.SetField(index.DependsOn) + conjunctionQuery.AddQuery(termQuery) + } + q.addQuery(conjunctionQuery) + return q +} + +func (q *manifestQuery) WithCreatedAfter(t time.Time) Query { + dateQuery := bleve.NewTermRangeQuery(t.Format(time.RFC3339Nano), "") + dateQuery.SetField(index.MetaCreatedAt) + q.addQuery(dateQuery) + return q +} + +func (q *manifestQuery) WithCreatedBefore(t time.Time) Query { + dateQuery := bleve.NewTermRangeQuery("", t.Format(time.RFC3339Nano)) + dateQuery.SetField(index.MetaCreatedAt) + q.addQuery(dateQuery) + return q +} + +func (q *manifestQuery) WithUpdatedAfter(t time.Time) Query { + dateQuery := bleve.NewTermRangeQuery(t.Format(time.RFC3339Nano), "") + dateQuery.SetField(index.MetaUpdatedAt) + q.addQuery(dateQuery) + return q +} + +func (q *manifestQuery) WithUpdatedBefore(t time.Time) Query { + dateQuery := bleve.NewTermRangeQuery("", t.Format(time.RFC3339Nano)) + dateQuery.SetField(index.MetaUpdatedAt) + q.addQuery(dateQuery) + return q +} + +func (q *manifestQuery) WithLastApplied(t time.Time) Query { + dateQuery := bleve.NewTermRangeQuery("", t.Format(time.RFC3339Nano)) + dateQuery.SetField(index.MetaLastApplied) + q.addQuery(dateQuery) + return q +} + +func (q *manifestQuery) addQuery(newQuery query.Query) { + if _, ok := q.bleveQuery.(*query.MatchAllQuery); ok { + q.bleveQuery = newQuery + } else { + q.bleveQuery = bleve.NewConjunctionQuery(q.bleveQuery, newQuery) + } +}