diff --git a/cmd/cli/apply.go b/cmd/cli/apply.go index 603e41f..84de9d8 100644 --- a/cmd/cli/apply.go +++ b/cmd/cli/apply.go @@ -8,7 +8,7 @@ import ( ) func init() { - applyCmd.Flags().StringP("file", "f", ".", "Path to manifest file") + applyCmd.Flags().StringP("file", "f", ".", "Path to manifests file") rootCmd.AddCommand(applyCmd) } @@ -27,16 +27,17 @@ var applyCmd = &cobra.Command{ ui.Printf("Applying manifests from: %s", file) ui.Spinner(true, "Loading manifests") - mans, err := loader.LoadManifestsFromDir(file) + loadedMans, _, err := loader.LoadManifests(file) if err != nil { ui.Spinner(false) ui.Errorf("Failed to load manifests: %s", err.Error()) return } + ui.Spinner(false) ui.Spinner(true, "Saving manifests...") - if err := store.SaveManifests(mans...); err != nil { + if err := store.Save(loadedMans...); err != nil { ui.Error("Failed to save manifests: " + err.Error()) ui.Spinner(false) return diff --git a/cmd/cli/cleanup.go b/cmd/cli/cleanup.go new file mode 100644 index 0000000..83050f5 --- /dev/null +++ b/cmd/cli/cleanup.go @@ -0,0 +1,77 @@ +package cli + +import ( + "fmt" + + "github.com/apiqube/cli/internal/core/store" + "github.com/apiqube/cli/ui" + "github.com/spf13/cobra" +) + +const keepVersionDefault = 5 + +var cleanupCmd = &cobra.Command{ + Use: "cleanup [ID]", + Short: "Cleanup old manifest versions by its id", + Long: fmt.Sprintf("Delete all versions of the manifest,"+ + "\nleaving only the latest specified."+ + "\nBy default, the last keep amount is %d", keepVersionDefault), + SilenceErrors: true, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + opts, err := parseCleanUpFlags(cmd, args) + if err != nil { + return err + } + + keep := opts.Keep + if keep <= 0 { + keep = keepVersionDefault + } + + ui.Spinner(true, "Cleaning up...") + if err = store.CleanupOldVersions(opts.ManifestID, keep); err != nil { + ui.Spinner(false) + ui.Errorf("Failed to cleanup old versions: %v", err) + } + + ui.Spinner(false) + ui.Successf("Successfully cleaned up %v to last %d versions", opts.ManifestID, keep) + return nil + }, +} + +func init() { + rootCmd.AddCommand(cleanupCmd) + cleanupCmd.Flags().IntP("keep", "k", keepVersionDefault, "Number of last versions to keep") +} + +type CleanUpOptions struct { + ManifestID string + Keep int +} + +func parseCleanUpFlags(cmd *cobra.Command, args []string) (*CleanUpOptions, error) { + opts := &CleanUpOptions{} + + if len(args) == 0 { + return nil, fmt.Errorf("manifest ID is required") + } + + opts.ManifestID = args[0] + var err error + var keep int + + if cmd.Flags().Changed("keep") { + keep, err = cmd.Flags().GetInt("keep") + if err != nil { + return nil, fmt.Errorf("invalid keep value: %w", err) + } + if keep < 1 { + return nil, fmt.Errorf("keep value must be positive") + } + opts.Keep = keep + } + + return opts, nil +} diff --git a/cmd/cli/rollback.go b/cmd/cli/rollback.go new file mode 100644 index 0000000..77deef2 --- /dev/null +++ b/cmd/cli/rollback.go @@ -0,0 +1,74 @@ +package cli + +import ( + "fmt" + + "github.com/apiqube/cli/internal/core/store" + "github.com/apiqube/cli/ui" + "github.com/spf13/cobra" +) + +var rollbackCmd = &cobra.Command{ + Use: "rollback [ID]", + Short: "Rollback to previous manifest version", + Long: fmt.Sprint("Rollback to specific version of manifest." + + "\nIf version is not specified, rolls back to previous one."), + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + opts, err := parseRollbackFlags(cmd, args) + if err != nil { + return + } + + targetVersion := opts.Version + if targetVersion <= 0 { + targetVersion = 1 + } + + ui.Spinner(true, "Rolling back") + if err = store.Rollback(opts.ManifestID, targetVersion); err != nil { + ui.Spinner(false, "Failed to rollback") + ui.Errorf("Error rolling back to previous version: %s", err) + return + } + + ui.Spinner(false) + ui.Successf("Successfully rolled back %s to version %d\n", opts.ManifestID, targetVersion) + }, +} + +func init() { + rollbackCmd.Flags().IntP("version", "v", 0, "Target version number (defaults to previous version)") + + rootCmd.AddCommand(rollbackCmd) +} + +type RollbackOptions struct { + ManifestID string + Version int +} + +func parseRollbackFlags(cmd *cobra.Command, args []string) (*RollbackOptions, error) { + opts := &RollbackOptions{} + + if len(args) == 0 { + return nil, fmt.Errorf("manifest ID is required") + } + + opts.ManifestID = args[0] + var err error + var ver int + + if cmd.Flags().Changed("version") { + ver, err = cmd.Flags().GetInt("version") + if err != nil { + return nil, fmt.Errorf("invalid version: %w", err) + } + if ver < 1 { + return nil, fmt.Errorf("version must be positive") + } + opts.Version = ver + } + + return opts, nil +} diff --git a/cmd/cli/root.go b/cmd/cli/root.go index 0c9fce7..31852d8 100644 --- a/cmd/cli/root.go +++ b/cmd/cli/root.go @@ -12,7 +12,3 @@ var rootCmd = &cobra.Command{ func Execute() { cobra.CheckErr(rootCmd.Execute()) } - -func init() { - rootCmd.AddCommand(versionCmd) -} diff --git a/cmd/cli/search.go b/cmd/cli/search.go new file mode 100644 index 0000000..8ef36ae --- /dev/null +++ b/cmd/cli/search.go @@ -0,0 +1,568 @@ +package cli + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/apiqube/cli/internal/core/manifests" + "github.com/apiqube/cli/internal/core/store" + "github.com/apiqube/cli/ui" + "github.com/spf13/cobra" + "gopkg.in/yaml.v3" +) + +var searchCmd = &cobra.Command{ + Use: "search", + Short: "Search for manifests using filters", + Long: fmt.Sprint("Search for manifests with powerful filtering options including exact/wildcard matching," + + "\ntime ranges, and output formatting"), + SilenceUsage: true, + SilenceErrors: true, + RunE: func(cmd *cobra.Command, args []string) error { + opts, err := parseSearchFlags(cmd, args) + if err != nil { + ui.Errorf("Failed to parse provided values: %v", err) + return err + } + + var manifests []manifests.Manifest + + if !opts.All && + !opts.flagsSet["name"] && + !opts.flagsSet["name-wildcard"] && + !opts.flagsSet["name-regex"] && + !opts.flagsSet["kind"] && + !opts.flagsSet["hash"] && + !opts.flagsSet["version"] && + !opts.flagsSet["namespace"] && + !opts.flagsSet["created-by"] && + !opts.flagsSet["used-by"] && + !opts.flagsSet["depends"] && + !opts.flagsSet["depends-all"] && + !opts.flagsSet["created-after"] && + !opts.flagsSet["created-before"] && + !opts.flagsSet["updated-after"] && + !opts.flagsSet["updated-before"] && + !opts.flagsSet["last-applied"] { + return fmt.Errorf("at least one search filter must be specified") + } + + if opts.flagsSet["all"] { + manifests, err = store.Load(store.LoadOptions{All: true}) + if err != nil { + ui.Errorf("Failed to loadmanifests: %v", err) + return nil + } + } else { + query := store.NewQuery() + + if opts.flagsSet["name"] { + query.WithExactName(opts.Name) + } else if opts.flagsSet["name-wildcard"] { + query.WithWildcardName(opts.NameWildcard) + } else if opts.flagsSet["name-regex"] { + query.WithRegexName(opts.NameRegex) + } + + if opts.flagsSet["namespace"] { + query.WithNamespace(opts.Namespace) + } + + if opts.flagsSet["kind"] { + query.WithKind(opts.Kind) + } + + if opts.flagsSet["version"] { + query.WithVersion(opts.Version) + } + + if opts.flagsSet["created-by"] { + query.WithCreatedBy(opts.CreatedBy) + } + + if opts.flagsSet["user-by"] { + query.WithUsedBy(opts.UsedBy) + } + + if opts.flagsSet["hash"] { + query.WithHashPrefix(opts.HashPrefix) + } + + if opts.flagsSet["depends"] { + query.WithDependencies(opts.DependsOn) + } else if opts.flagsSet["depends-all"] { + query.WithAllDependencies(opts.DependsOnAll) + } + + if opts.flagsSet["created-after"] { + query.WithCreatedAfter(opts.CreatedAfter) + } + + if opts.flagsSet["created-before"] { + query.WithCreatedBefore(opts.CreatedBefore) + } + + if opts.flagsSet["updated-after"] { + query.WithUpdatedAfter(opts.UpdatedAfter) + } + + if opts.flagsSet["updated-before"] { + query.WithUpdatedBefore(opts.UpdatedBefore) + } + + if opts.flagsSet["last-applied"] { + query.WithLastApplied(opts.LastApplied) + } + + manifests, err = store.Search(query) + if err != nil { + ui.Errorf("Failed to search manifests: %v", err) + return nil + } + } + + if len(manifests) == 0 { + ui.Warning("No manifests found matching the criteria") + return nil + } + + ui.Infof("Found %d manifests", len(manifests)) + + if len(opts.SortBy) > 0 { + sortManifests(manifests, opts.SortBy) + } + + ui.Spinner(true, "Prepare answer...") + + if opts.Output { + if err := outputManifests(manifests, opts); err != nil { + ui.Spinner(false) + ui.Errorf("Failed to output manifests: %v", err) + return nil + } + } else { + displayResults(manifests) + } + + ui.Spinner(false, "Complete") + + return nil + }, +} + +func init() { + searchCmd.Flags().BoolP("all", "a", false, "Get all manifests") + + searchCmd.Flags().StringP("name", "n", "", "Search manifest by name (exact match)") + searchCmd.Flags().StringP("name-wildcard", "W", "", "Search manifest by wildcard pattern (e.g. '*name*')") + searchCmd.Flags().StringP("name-regex", "R", "", "Search manifest by regex pattern") + + searchCmd.Flags().StringP("namespace", "s", "", "Search manifests by namespace") + searchCmd.Flags().StringP("kind", "k", "", "Search manifests by kind") + searchCmd.Flags().IntP("version", "v", 0, "Search manifests by version") + searchCmd.Flags().String("created-by", "", "Filter by exact creator username") + searchCmd.Flags().String("used-by", "", "Filter by exact user who applied") + + searchCmd.Flags().StringP("hash", "H", "", "Search manifests by hash prefix (min 5 chars)") + searchCmd.Flags().StringSliceP("depends", "d", []string{}, "Search manifests by dependencies (comma separated)") + searchCmd.Flags().StringSliceP("depends-all", "D", []string{}, "Search manifests by all dependencies (comma separated)") + + searchCmd.Flags().String("created-after", "", "Search manifests created after date (YYYY-MM-DD or duration like 1h30m)") + searchCmd.Flags().String("created-before", "", "Search manifests created before date/duration") + searchCmd.Flags().String("updated-after", "", "Search manifests updated after date/duration") + searchCmd.Flags().String("updated-before", "", "Search manifests updated before date/duration") + searchCmd.Flags().String("last-applied", "", "Search manifests by last applied date/duration") + + searchCmd.Flags().BoolP("output", "o", false, "Make output after searching") + searchCmd.Flags().String("output-path", "", "Output path for results (default: current directory)") + searchCmd.Flags().String("output-mode", "separate", "Output mode (combined|separate)") + searchCmd.Flags().String("output-format", "yaml", "File format for output (yaml|json)") + + searchCmd.Flags().StringSlice("sort", []string{}, "Sort by fields (e.g. --sort=kind,-name)") + + rootCmd.AddCommand(searchCmd) +} + +type SearchOptions struct { + All bool + + Name string + NameWildcard string + NameRegex string + + Namespace string + Kind string + Version int + CreatedBy string + UsedBy string + + HashPrefix string + DependsOn []string + DependsOnAll []string + + CreatedAfter time.Time + CreatedBefore time.Time + UpdatedAfter time.Time + UpdatedBefore time.Time + LastApplied time.Time + IsRelativeTime bool + + Output bool + OutputPath string + OutputMode string // combined | separate + OutputFormat string // yaml | json + + SortBy []string + + flagsSet map[string]bool +} + +func parseSearchFlags(cmd *cobra.Command, _ []string) (*SearchOptions, error) { + opts := &SearchOptions{ + flagsSet: make(map[string]bool), + } + + markFlag := func(name string) bool { + if cmd.Flags().Changed(name) { + opts.flagsSet[name] = true + return true + } + return false + } + + if markFlag("all") { + opts.All, _ = cmd.Flags().GetBool("all") + } + + if markFlag("name") { + opts.Name, _ = cmd.Flags().GetString("name") + } + if markFlag("name-wildcard") { + opts.NameWildcard, _ = cmd.Flags().GetString("name-wildcard") + } + if markFlag("name-regex") { + opts.NameRegex, _ = cmd.Flags().GetString("name-regex") + } + + if opts.flagsSet["name"] && (opts.flagsSet["name-wildcard"] || opts.flagsSet["name-regex"]) { + return nil, fmt.Errorf("cannot use exact name filter with wildcard/regex filters") + } + + if markFlag("namespace") { + opts.Namespace, _ = cmd.Flags().GetString("namespace") + } + if markFlag("kind") { + opts.Kind, _ = cmd.Flags().GetString("kind") + } + if markFlag("version") { + opts.Version, _ = cmd.Flags().GetInt("version") + } + if markFlag("created-by") { + opts.CreatedBy, _ = cmd.Flags().GetString("created-by") + } + if markFlag("used-by") { + opts.UsedBy, _ = cmd.Flags().GetString("used-by") + } + + if markFlag("hash") { + opts.HashPrefix, _ = cmd.Flags().GetString("hash") + if len(opts.HashPrefix) < 5 { + return nil, fmt.Errorf("hash prefix must be at least 5 characters") + } + } + if markFlag("depends") { + opts.DependsOn, _ = cmd.Flags().GetStringSlice("depends") + } else if markFlag("depends-all") { + opts.DependsOnAll, _ = cmd.Flags().GetStringSlice("depends-all") + } + + timeFilters := map[string]*time.Time{ + "created-after": &opts.CreatedAfter, + "created-before": &opts.CreatedBefore, + "updated-after": &opts.UpdatedAfter, + "updated-before": &opts.UpdatedBefore, + "last-applied": &opts.LastApplied, + } + + for flag, target := range timeFilters { + if markFlag(flag) { + val, _ := cmd.Flags().GetString(flag) + if t, err := parseTimeOrDuration(val); err == nil { + *target = t + opts.IsRelativeTime = isDuration(val) + } else { + return nil, fmt.Errorf("invalid %s value: %w", flag, err) + } + } + } + + if markFlag("output") { + opts.Output, _ = cmd.Flags().GetBool("output") + if opts.Output { + if markFlag("output-path") { + opts.OutputPath, _ = cmd.Flags().GetString("output-path") + } + if opts.OutputPath == "" { + opts.OutputPath = "." + } + if markFlag("output-mode") { + opts.OutputMode, _ = cmd.Flags().GetString("output-mode") + if opts.OutputMode != "combined" && opts.OutputMode != "separate" { + return nil, fmt.Errorf("invalid output mode, must be 'combined' or 'separate'") + } + } + if opts.OutputMode == "" { + opts.OutputMode = "separate" + } + if markFlag("output-format") { + opts.OutputFormat, _ = cmd.Flags().GetString("output-format") + if opts.OutputFormat != "yaml" && opts.OutputFormat != "json" { + return nil, fmt.Errorf("invalid output format, must be 'yaml' or 'json'") + } + } + if opts.OutputFormat == "" { + opts.OutputFormat = "yaml" + } + } + } + + if markFlag("sort") { + opts.SortBy, _ = cmd.Flags().GetStringSlice("sort") + } + + return opts, nil +} + +func parseTimeOrDuration(val string) (time.Time, error) { + if duration, err := time.ParseDuration(val); err == nil { + return time.Now().Add(-duration), nil + } + + if t, err := time.Parse("2006-01-02", val); err == nil { + return t, nil + } + + if t, err := time.Parse(time.RFC3339, val); err == nil { + return t, nil + } + + return time.Time{}, fmt.Errorf("invalid time format") +} + +func isDuration(val string) bool { + _, err := time.ParseDuration(val) + return err == nil +} + +func outputManifests(manifests []manifests.Manifest, opts *SearchOptions) error { + if err := os.MkdirAll(opts.OutputPath, 0o755); err != nil { + return fmt.Errorf("failed to create output directory: %w", err) + } + + if opts.OutputMode == "combined" { + filename := filepath.Join(opts.OutputPath, fmt.Sprintf("manifests.%s", opts.OutputFormat)) + return writeCombinedFile(filename, manifests, opts.OutputFormat) + } else { + for _, m := range manifests { + filename := filepath.Join(opts.OutputPath, fmt.Sprintf("%s.%s", m.GetID(), opts.OutputFormat)) + if err := writeSingleFile(filename, m, opts.OutputFormat); err != nil { + return err + } + } + } + + return nil +} + +func writeCombinedFile(filename string, manifests []manifests.Manifest, format string) error { + if len(manifests) == 0 { + return fmt.Errorf("no manifests to write") + } + + file, err := os.Create(filename) + if err != nil { + return fmt.Errorf("failed to create file %s: %w", filename, err) + } + defer func() { + _ = file.Close() + }() + + switch strings.ToLower(format) { + case "yaml": + encoder := yaml.NewEncoder(file) + for i, m := range manifests { + if i > 0 { + if _, err = file.WriteString("---\n"); err != nil { + return fmt.Errorf("failed to write YAML manifest: %w", err) + } + } + if err = encoder.Encode(m); err != nil { + return fmt.Errorf("failed to encode manifest %d: %w", i+1, err) + } + } + case "json": + if _, err = file.WriteString("[\n"); err != nil { + return fmt.Errorf("failed to write JSON manifest: %w", err) + } + + encoder := json.NewEncoder(file) + encoder.SetIndent("", " ") + + for i, m := range manifests { + if i > 0 { + if _, err = file.WriteString(",\n"); err != nil { + return fmt.Errorf("failed to write YAML manifest: %w", err) + } + } + if err = encoder.Encode(m); err != nil { + return fmt.Errorf("failed to encode manifest %d: %w", i+1, err) + } + } + if _, err = file.WriteString("\n]"); err != nil { + return fmt.Errorf("failed to write JSON manifest: %w", err) + } + default: + return fmt.Errorf("unsupported format: %s", format) + } + + ui.Successf("Successfully wrote %d manifests to %s", len(manifests), filename) + return nil +} + +func writeSingleFile(filename string, manifest manifests.Manifest, format string) error { + file, err := os.Create(filename) + if err != nil { + return fmt.Errorf("failed to create file %s: %w", filename, err) + } + defer func() { + _ = file.Close() + }() + + switch strings.ToLower(format) { + case "yaml": + encoder := yaml.NewEncoder(file) + if err = encoder.Encode(manifest); err != nil { + return fmt.Errorf("failed to encode manifest: %w", err) + } + case "json": + encoder := json.NewEncoder(file) + encoder.SetIndent("", " ") + if err = encoder.Encode(manifest); err != nil { + return fmt.Errorf("failed to encode manifest: %w", err) + } + default: + return fmt.Errorf("unsupported format: %s", format) + } + + ui.Successf("Successfully wrote manifest %s to %s", manifest.GetID(), filename) + return nil +} + +func sortManifests(manifests []manifests.Manifest, fields []string) { + sort.Slice(manifests, func(i, j int) bool { + for _, field := range fields { + desc := false + if strings.HasPrefix(field, "-") { + desc = true + field = field[1:] + } + + switch field { + case "id": + if manifests[i].GetID() != manifests[j].GetID() { + if desc { + return manifests[i].GetID() > manifests[j].GetID() + } + return manifests[i].GetID() < manifests[j].GetID() + } + case "name": + if manifests[i].GetName() != manifests[j].GetName() { + if desc { + return manifests[i].GetName() > manifests[j].GetName() + } + return manifests[i].GetName() < manifests[j].GetName() + } + case "kind": + if manifests[i].GetKind() != manifests[j].GetKind() { + if desc { + return manifests[i].GetKind() > manifests[j].GetKind() + } + return manifests[i].GetKind() < manifests[j].GetKind() + } + case "namespace": + if manifests[i].GetNamespace() != manifests[j].GetNamespace() { + if desc { + return manifests[i].GetNamespace() > manifests[j].GetNamespace() + } + return manifests[i].GetNamespace() < manifests[j].GetNamespace() + } + case "version": + if desc { + return manifests[i].GetMeta().GetVersion() > manifests[j].GetMeta().GetVersion() + } + return manifests[i].GetMeta().GetVersion() < manifests[j].GetMeta().GetVersion() + case "created": + if desc { + return manifests[i].GetMeta().GetCreatedAt().After(manifests[j].GetMeta().GetCreatedAt()) + } + return manifests[i].GetMeta().GetCreatedAt().Before(manifests[j].GetMeta().GetCreatedAt()) + case "updated": + if desc { + return manifests[i].GetMeta().GetUpdatedAt().After(manifests[j].GetMeta().GetUpdatedAt()) + } + return manifests[i].GetMeta().GetUpdatedAt().Before(manifests[j].GetMeta().GetUpdatedAt()) + case "last": + if desc { + return manifests[i].GetMeta().GetLastApplied().After(manifests[j].GetMeta().GetLastApplied()) + } + return manifests[i].GetMeta().GetLastApplied().Before(manifests[j].GetMeta().GetLastApplied()) + } + } + return false + }) +} + +func displayResults(manifests []manifests.Manifest) { + headers := []string{ + "#", + "Hash", + "Kind", + "Name", + "Namespace", + "Version", + "Created", + "Updated", + "Last Updated", + } + + var rows [][]string + for i, m := range manifests { + meta := m.GetMeta() + row := []string{ + fmt.Sprint(i + 1), + shortHash(meta.GetHash()), + m.GetKind(), + m.GetName(), + m.GetNamespace(), + fmt.Sprint(meta.GetVersion()), + meta.GetCreatedAt().Format(time.RFC3339), + meta.GetUpdatedAt().Format(time.RFC3339), + meta.GetLastApplied().Format(time.RFC3339), + } + rows = append(rows, row) + } + + ui.Table(headers, rows) +} + +func shortHash(fullHash string) string { + if len(fullHash) > 8 { + return fullHash[:8] + } + return fullHash +} diff --git a/cmd/cli/version.go b/cmd/cli/version.go index 38acbc7..c12d00d 100644 --- a/cmd/cli/version.go +++ b/cmd/cli/version.go @@ -14,6 +14,10 @@ var versionCmd = &cobra.Command{ SilenceUsage: true, SilenceErrors: true, Run: func(cmd *cobra.Command, args []string) { - fmt.Println("Qube CLI Version: ", version) + fmt.Println("Qube CLI", version) }, } + +func init() { + rootCmd.AddCommand(versionCmd) +} diff --git a/examples/simple/http_test.yaml b/examples/simple/http_test.yaml index a2d83cd..d0d78dc 100644 --- a/examples/simple/http_test.yaml +++ b/examples/simple/http_test.yaml @@ -34,7 +34,7 @@ spec: method: GET endpoint: /users/{id} expected: - code: 404 + code: 201 message: "User not found" dependsOn: diff --git a/examples/simple/server.yaml b/examples/simple/server.yaml index 9084ef8..0f60e8d 100644 --- a/examples/simple/server.yaml +++ b/examples/simple/server.yaml @@ -6,6 +6,6 @@ metadata: name: simple-server spec: - baseUrl: "http://localhost:8080" + baseUrl: "http://localhost:8081" headers: Content-Type: application/json \ No newline at end of file diff --git a/examples/values/values.yaml b/examples/values/values.yaml index 9b839c2..b8172c5 100644 --- a/examples/values/values.yaml +++ b/examples/values/values.yaml @@ -6,7 +6,7 @@ metadata: spec: users: - username: ["Max", "Carl", "John", "Alex"] + username: ["Max", "Carl", "John", "Alex", "Uli"] email: - "email_1@gmail.com" - "email_2@mail.com" diff --git a/go.mod b/go.mod index 17d77b7..cb3e4e8 100644 --- a/go.mod +++ b/go.mod @@ -67,7 +67,7 @@ require ( go.opentelemetry.io/otel/trace v1.35.0 // indirect golang.org/x/net v0.38.0 // indirect golang.org/x/sync v0.13.0 // indirect - golang.org/x/sys v0.32.0 // indirect + golang.org/x/sys v0.33.0 // indirect golang.org/x/text v0.23.0 // indirect google.golang.org/protobuf v1.36.6 // indirect ) diff --git a/go.sum b/go.sum index e9d6318..1bd3d90 100644 --- a/go.sum +++ b/go.sum @@ -152,8 +152,8 @@ golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= -golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/core/manifests/hash/hash.go b/internal/core/manifests/hash/hash.go index c58373d..75c1446 100644 --- a/internal/core/manifests/hash/hash.go +++ b/internal/core/manifests/hash/hash.go @@ -25,3 +25,11 @@ func CalculateHashWithPath(filePath string, content []byte) (string, error) { return hash, nil } + +func CalculateHashWithContent(content []byte) (string, error) { + hasher := sha256.New() + hasher.Write(content) + hash := hex.EncodeToString(hasher.Sum(nil)) + + return hash, nil +} diff --git a/internal/core/manifests/index/index.go b/internal/core/manifests/index/index.go index 1ddfb07..f4e5853 100644 --- a/internal/core/manifests/index/index.go +++ b/internal/core/manifests/index/index.go @@ -1,6 +1,7 @@ package index const ( + ID = "id" Version = "version" Kind = "kind" Name = "name" @@ -10,6 +11,8 @@ const ( const ( MetaHash = "meta.hash" + MetaVersion = "meta.version" + MetaIsCurrent = "meta.isCurrent" MetaCreatedAt = "meta.createdAt" MetaCreatedBy = "meta.createdBy" MetaUpdatedAt = "meta.updatedAt" diff --git a/internal/core/manifests/interface.go b/internal/core/manifests/interface.go index 61d75f9..7e72f8f 100644 --- a/internal/core/manifests/interface.go +++ b/internal/core/manifests/interface.go @@ -27,16 +27,13 @@ type Manifest interface { GetName() string GetNamespace() string Index() any + GetMeta() Meta } type Dependencies interface { GetDependsOn() []string } -type MetaTable interface { - GetMeta() Meta -} - type Meta interface { GetHash() string SetHash(hash string) @@ -45,6 +42,9 @@ type Meta interface { SetVersion(version uint8) IncVersion() + GetIsCurrent() bool + SetIsCurrent(isCurrent bool) + GetCreatedAt() time.Time SetCreatedAt(createdAt time.Time) diff --git a/internal/core/manifests/kinds/meta.go b/internal/core/manifests/kinds/meta.go index d79ece6..6fe19f0 100644 --- a/internal/core/manifests/kinds/meta.go +++ b/internal/core/manifests/kinds/meta.go @@ -21,6 +21,7 @@ func DefaultMeta() *Meta { return &Meta{ Hash: "", Version: 1, + IsCurrent: true, CreatedAt: time.Now(), CreatedBy: name, UpdatedAt: time.Now(), @@ -35,6 +36,7 @@ var _ manifests.Meta = (*Meta)(nil) type Meta struct { Hash string `yaml:"-" json:"hash"` Version uint8 `yaml:"-" json:"version"` + IsCurrent bool `yaml:"-" json:"isCurrent"` CreatedAt time.Time `yaml:"-" json:"createdAt"` CreatedBy string `yaml:"-" json:"createdBy"` UpdatedAt time.Time `yaml:"-" json:"updatedAt"` @@ -65,6 +67,14 @@ func (m *Meta) IncVersion() { } } +func (m *Meta) GetIsCurrent() bool { + return m.IsCurrent +} + +func (m *Meta) SetIsCurrent(isCurrent bool) { + m.IsCurrent = isCurrent +} + func (m *Meta) GetCreatedAt() time.Time { return m.CreatedAt } diff --git a/internal/core/manifests/kinds/servers/server.go b/internal/core/manifests/kinds/servers/server.go index 3c61d96..9981620 100644 --- a/internal/core/manifests/kinds/servers/server.go +++ b/internal/core/manifests/kinds/servers/server.go @@ -10,7 +10,6 @@ import ( var ( _ manifests.Manifest = (*Server)(nil) - _ manifests.MetaTable = (*Server)(nil) _ manifests.Defaultable = (*Server)(nil) _ manifests.Prepare = (*Server)(nil) ) @@ -44,12 +43,15 @@ func (s *Server) GetNamespace() string { func (s *Server) Index() any { return map[string]any{ + index.ID: s.GetID(), index.Version: float64(s.Version), index.Kind: s.Kind, index.Name: s.Name, index.Namespace: s.Namespace, index.MetaHash: s.Meta.Hash, + index.MetaVersion: float64(s.Meta.Version), + index.MetaIsCurrent: s.Meta.IsCurrent, index.MetaCreatedAt: s.Meta.CreatedAt.Format(time.RFC3339Nano), index.MetaCreatedBy: s.Meta.CreatedBy, index.MetaUpdatedAt: s.Meta.UpdatedAt.Format(time.RFC3339Nano), diff --git a/internal/core/manifests/kinds/services/service.go b/internal/core/manifests/kinds/services/service.go index 3d58079..7b0943a 100644 --- a/internal/core/manifests/kinds/services/service.go +++ b/internal/core/manifests/kinds/services/service.go @@ -11,7 +11,6 @@ import ( var ( _ manifests.Manifest = (*Service)(nil) _ manifests.Dependencies = (*Service)(nil) - _ manifests.MetaTable = (*Service)(nil) _ manifests.Defaultable = (*Service)(nil) _ manifests.Prepare = (*Service)(nil) ) @@ -49,6 +48,7 @@ func (s *Service) GetDependsOn() []string { func (s *Service) Index() any { return map[string]any{ + index.ID: s.GetID(), index.Version: float64(s.Version), index.Kind: s.Kind, index.Name: s.Name, @@ -56,6 +56,8 @@ func (s *Service) Index() any { index.DependsOn: s.DependsOn, index.MetaHash: s.Meta.Hash, + index.MetaVersion: float64(s.Meta.Version), + index.MetaIsCurrent: s.Meta.IsCurrent, index.MetaCreatedAt: s.Meta.CreatedAt.Format(time.RFC3339Nano), index.MetaCreatedBy: s.Meta.CreatedBy, index.MetaUpdatedAt: s.Meta.UpdatedAt.Format(time.RFC3339Nano), diff --git a/internal/core/manifests/kinds/system/plan/plan.go b/internal/core/manifests/kinds/system/plan/plan.go index 563c157..b62b38c 100644 --- a/internal/core/manifests/kinds/system/plan/plan.go +++ b/internal/core/manifests/kinds/system/plan/plan.go @@ -10,7 +10,6 @@ import ( var ( _ manifests.Manifest = (*Plan)(nil) - _ manifests.MetaTable = (*Plan)(nil) _ manifests.Defaultable = (*Plan)(nil) _ manifests.Prepare = (*Plan)(nil) ) @@ -59,12 +58,15 @@ func (p *Plan) GetNamespace() string { func (p *Plan) Index() any { return map[string]any{ + index.ID: p.GetID(), index.Version: float64(p.Version), index.Kind: p.Kind, index.Name: p.Name, index.Namespace: p.Namespace, index.MetaHash: p.Meta.Hash, + index.MetaVersion: float64(p.Meta.Version), + index.MetaIsCurrent: p.Meta.IsCurrent, index.MetaCreatedAt: p.Meta.CreatedAt.Format(time.RFC3339Nano), index.MetaCreatedBy: p.Meta.CreatedBy, index.MetaUpdatedAt: p.Meta.UpdatedAt.Format(time.RFC3339Nano), diff --git a/internal/core/manifests/kinds/tests/api/http.go b/internal/core/manifests/kinds/tests/api/http.go index 9f495f2..0018e20 100644 --- a/internal/core/manifests/kinds/tests/api/http.go +++ b/internal/core/manifests/kinds/tests/api/http.go @@ -15,7 +15,6 @@ import ( var ( _ manifests.Manifest = (*Http)(nil) _ manifests.Dependencies = (*Http)(nil) - _ manifests.MetaTable = (*Http)(nil) _ manifests.Defaultable = (*Http)(nil) _ manifests.Prepare = (*Http)(nil) ) @@ -61,6 +60,8 @@ func (h *Http) Index() any { index.DependsOn: h.DependsOn, index.MetaHash: h.Meta.Hash, + index.MetaVersion: float64(h.Meta.Version), + index.MetaIsCurrent: h.Meta.IsCurrent, index.MetaCreatedAt: h.Meta.CreatedAt.Format(time.RFC3339Nano), index.MetaCreatedBy: h.Meta.CreatedBy, index.MetaUpdatedAt: h.Meta.UpdatedAt.Format(time.RFC3339Nano), diff --git a/internal/core/manifests/kinds/tests/load/http.go b/internal/core/manifests/kinds/tests/load/http.go index 6534828..87a87e3 100644 --- a/internal/core/manifests/kinds/tests/load/http.go +++ b/internal/core/manifests/kinds/tests/load/http.go @@ -13,7 +13,6 @@ import ( var ( _ manifests.Manifest = (*Http)(nil) _ manifests.Dependencies = (*Http)(nil) - _ manifests.MetaTable = (*Http)(nil) _ manifests.Defaultable = (*Http)(nil) _ manifests.Prepare = (*Http)(nil) ) @@ -77,6 +76,7 @@ func (h *Http) GetNamespace() string { func (h *Http) Index() any { return map[string]any{ + index.ID: h.GetID(), index.Version: float64(h.Version), index.Kind: h.Kind, index.Name: h.Name, @@ -84,6 +84,8 @@ func (h *Http) Index() any { index.DependsOn: h.DependsOn, index.MetaHash: h.Meta.Hash, + index.MetaVersion: float64(h.Meta.Version), + index.MetaIsCurrent: h.Meta.IsCurrent, index.MetaCreatedAt: h.Meta.CreatedAt.Format(time.RFC3339Nano), index.MetaCreatedBy: h.Meta.CreatedBy, index.MetaUpdatedAt: h.Meta.UpdatedAt.Format(time.RFC3339Nano), diff --git a/internal/core/manifests/kinds/values/values.go b/internal/core/manifests/kinds/values/values.go index 22fe819..4b70f11 100644 --- a/internal/core/manifests/kinds/values/values.go +++ b/internal/core/manifests/kinds/values/values.go @@ -11,7 +11,6 @@ import ( var ( _ manifests.Manifest = (*Values)(nil) _ manifests.Defaultable = (*Values)(nil) - _ manifests.MetaTable = (*Values)(nil) _ manifests.Prepare = (*Values)(nil) ) @@ -19,16 +18,12 @@ type Values struct { kinds.BaseManifest `yaml:",inline" json:",inline"` Spec struct { - Content `yaml:",inline" json:",inline"` + Data map[string]any `yaml:",inline" json:",inline"` } `yaml:"spec" valid:"required"` Meta *kinds.Meta `yaml:"-" json:"meta"` } -type Content struct { - Values map[string]any `yaml:",inline" json:",inline"` -} - func (v *Values) GetID() string { return kinds.FormManifestID(v.Namespace, v.Kind, v.Name) } @@ -47,12 +42,15 @@ func (v *Values) GetNamespace() string { func (v *Values) Index() any { return map[string]any{ + index.ID: v.GetID(), index.Version: float64(v.Version), index.Kind: v.Kind, index.Name: v.Name, index.Namespace: v.Namespace, index.MetaHash: v.Meta.Hash, + index.MetaVersion: float64(v.Meta.Version), + index.MetaIsCurrent: v.Meta.IsCurrent, index.MetaCreatedAt: v.Meta.CreatedAt.Format(time.RFC3339Nano), index.MetaCreatedBy: v.Meta.CreatedBy, index.MetaUpdatedAt: v.Meta.UpdatedAt.Format(time.RFC3339Nano), diff --git a/internal/core/manifests/loader/loader.go b/internal/core/manifests/loader/loader.go index 00a4a35..69c078f 100644 --- a/internal/core/manifests/loader/loader.go +++ b/internal/core/manifests/loader/loader.go @@ -4,9 +4,12 @@ import ( "fmt" "os" "path/filepath" + "sort" "strings" "time" + "gopkg.in/yaml.v3" + "github.com/apiqube/cli/internal/core/manifests/hash" "github.com/apiqube/cli/internal/core/manifests/parsing" "github.com/apiqube/cli/internal/core/store" @@ -15,90 +18,129 @@ import ( "github.com/apiqube/cli/internal/core/manifests" ) -func LoadManifestsFromDir(dir string) ([]manifests.Manifest, error) { - files, err := os.ReadDir(dir) +func LoadManifests(path string) (new []manifests.Manifest, cached []manifests.Manifest, err error) { + if isYAMLFile(path) { + return processSingleFile(path) + } + + files, err := os.ReadDir(path) if err != nil { - return nil, fmt.Errorf("failed to read directory: %w", err) + return nil, nil, fmt.Errorf("failed to read directory: %w", err) } - var ( - manifestsList []manifests.Manifest - manifestsSet = make(map[string]struct{}) - newCounter int - ) + manifestsSet := make(map[string]struct{}) + newCount, cachedCount := 0, 0 for _, file := range files { - if file.IsDir() || (!strings.HasSuffix(file.Name(), ".yaml") && !strings.HasSuffix(file.Name(), ".yml")) { + if file.IsDir() || !isYAMLFile(file.Name()) { continue } - var content []byte - filePath := filepath.Join(dir, file.Name()) - content, err = os.ReadFile(filePath) + filePath := filepath.Join(path, file.Name()) + + var fileNew, fileCached []manifests.Manifest + fileNew, fileCached, err = processFile(filePath, manifestsSet) if err != nil { - return nil, fmt.Errorf("error reading file %s: %w", filePath, err) + return nil, nil, err } - var fileHash string - fileHash, err = hash.CalculateHashWithPath(filePath, content) + new = append(new, fileNew...) + cached = append(cached, fileCached...) + newCount += len(fileNew) + cachedCount += len(fileCached) + } + + logResults(newCount, cachedCount) + return new, cached, nil +} + +func isYAMLFile(path string) bool { + return strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml") +} + +func processSingleFile(filePath string) (new []manifests.Manifest, cached []manifests.Manifest, err error) { + return processFile(filePath, make(map[string]struct{})) +} + +func processFile(filePath string, manifestsSet map[string]struct{}) (new []manifests.Manifest, cached []manifests.Manifest, err error) { + content, err := os.ReadFile(filePath) + if err != nil { + return nil, nil, fmt.Errorf("error reading file %s: %w", filePath, err) + } + + parsedManifests, err := parsing.ParseManifestsAsYAML(content) + if err != nil { + return nil, nil, fmt.Errorf("in file %s: %w", filepath.Base(filePath), err) + } + + now := time.Now() + var newManifests, cachedManifests []manifests.Manifest + + for _, m := range parsedManifests { + manifestID := m.GetID() + + var normalized []byte + normalized, err = normalizeYAML(m) + if err != nil { + return nil, nil, fmt.Errorf("failed to normalize manifest %s: %w", manifestID, err) + } + var manifestHash string + manifestHash, err = hash.CalculateHashWithContent(normalized) if err != nil { - return nil, fmt.Errorf("failed to calculate hash for %s: %w", filePath, err) + return nil, nil, fmt.Errorf("failed to calculate hash for manifest %s: %w", manifestID, err) } + var loadedManifests []manifests.Manifest var existingManifest manifests.Manifest - existingManifest, err = store.FindManifestByHash(fileHash) - if err != nil && !strings.Contains(err.Error(), "no matching manifest found") { - return nil, fmt.Errorf("failed to check manifest existence: %w", err) + + loadedManifests, err = store.Load(store.LoadOptions{Hash: manifestHash}) + if err != nil && !isNotFoundError(err) { + return nil, nil, fmt.Errorf("failed to check manifest existence: %w", err) + } + + if len(loadedManifests) > 0 { + existingManifest = loadedManifests[0] } if existingManifest != nil { if _, exists := manifestsSet[existingManifest.GetID()]; !exists { manifestsSet[existingManifest.GetID()] = struct{}{} - manifestsList = append(manifestsList, existingManifest) + ui.Infof("Manifest %s unchanged (%s) - using cached version", + existingManifest.GetID(), shortHash(manifestHash)) + cachedManifests = append(cachedManifests, existingManifest) } - - ui.Infof("Manifest file %s unchanged (%s) - using cached version", file.Name(), shortHash(fileHash)) continue } - var parsedManifests []manifests.Manifest - parsedManifests, err = parsing.ParseManifestsAsYAML(content) - if err != nil { - return nil, fmt.Errorf("in file %s: %w", file.Name(), err) + if _, exists := manifestsSet[manifestID]; exists { + ui.Warningf("Duplicate manifest ID: %s (from %s)", manifestID, filepath.Base(filePath)) + continue } - for _, m := range parsedManifests { - manifestID := m.GetID() + meta := m.GetMeta() + meta.SetHash(manifestHash) + meta.SetVersion(1) + meta.SetCreatedAt(now) + meta.SetUpdatedAt(now) - if _, exists := manifestsSet[manifestID]; exists { - ui.Warningf("Duplicate manifest ID: %s (from %s)", manifestID, file.Name()) - continue - } - - if metaTable, ok := m.(manifests.MetaTable); ok { - meta := metaTable.GetMeta() - meta.SetHash(fileHash) - meta.SetVersion(1) - now := time.Now() - meta.SetCreatedAt(now) - meta.SetUpdatedAt(now) - } + manifestsSet[manifestID] = struct{}{} + newManifests = append(newManifests, m) + ui.Successf("New manifest added: %s (h: %s)", manifestID, shortHash(manifestHash)) + } - manifestsSet[manifestID] = struct{}{} - manifestsList = append(manifestsList, m) - newCounter++ + return newManifests, cachedManifests, nil +} - ui.Successf("New manifest added: %s (h: %s)", manifestID, shortHash(fileHash)) - } +func logResults(cachedCount, newCount int) { + if cachedCount > 0 { + ui.Infof("Loaded %d cached manifests", cachedCount) } - - if newCounter == 0 { - ui.Info("No new manifests found in directory") - } else { - ui.Infof("Loaded %d new manifests", newCounter) + if newCount > 0 { + ui.Infof("Loaded %d new manifests", newCount) + } + if cachedCount == 0 && newCount == 0 { + ui.Info("No manifests found") } - - return manifestsList, nil } func shortHash(fullHash string) string { @@ -107,3 +149,44 @@ func shortHash(fullHash string) string { } return fullHash } + +func normalizeYAML(m manifests.Manifest) ([]byte, error) { + data, err := yaml.Marshal(m) + if err != nil { + return nil, err + } + + var raw map[string]interface{} + if err = yaml.Unmarshal(data, &raw); err != nil { + return nil, err + } + + sorted := sortMapKeys(raw) + + return yaml.Marshal(sorted) +} + +func sortMapKeys(m map[string]interface{}) map[string]interface{} { + res := make(map[string]interface{}) + keys := make([]string, 0, len(m)) + + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + + for _, k := range keys { + if nested, ok := m[k].(map[string]interface{}); ok { + res[k] = sortMapKeys(nested) + } else { + res[k] = m[k] + } + } + + return res +} + +func isNotFoundError(err error) bool { + return err != nil && (strings.Contains(err.Error(), "no matching manifest found") || + strings.Contains(err.Error(), "not found")) +} diff --git a/internal/core/store/db.go b/internal/core/store/db.go index fef57cb..e36316d 100644 --- a/internal/core/store/db.go +++ b/internal/core/store/db.go @@ -4,26 +4,41 @@ import ( "encoding/json" "errors" "fmt" + "math" "os" "path/filepath" + "sort" + "strconv" + "strings" "time" - "github.com/apiqube/cli/ui" - "github.com/apiqube/cli/internal/core/manifests/index" - "github.com/apiqube/cli/internal/core/manifests/parsing" - bleceQuery "github.com/blevesearch/bleve/v2/search/query" "github.com/adrg/xdg" "github.com/apiqube/cli/internal/core/manifests" + "github.com/apiqube/cli/internal/core/manifests/parsing" "github.com/blevesearch/bleve/v2" "github.com/dgraph-io/badger/v4" ) const ( - StorageDirPath = "ApiQube/Storage" + StorageDirPath = "ApiQube/storage" +) + +const ( + MaxManifestVersion = math.MaxUint8 - 230 ) +type LoadOptions struct { + IDs []string + Versions map[string]int // id -> version + Hash string + Latest bool + All bool + SortBy []string + Limit int +} + type Storage struct { db *badger.DB index bleve.Index @@ -63,354 +78,542 @@ func NewStorage() (*Storage, error) { }, nil } -func (s *Storage) SaveManifests(mans ...manifests.Manifest) error { - var err error - err = instance.db.Update(func(txn *badger.Txn) error { - var data []byte - +func (s *Storage) Save(mans ...manifests.Manifest) error { + return instance.db.Update(func(txn *badger.Txn) error { for _, m := range mans { - data, err = json.Marshal(m) + currentVer, err := s.getCurrentVersion(txn, m.GetID()) + if err != nil { + return fmt.Errorf("version check failed: %v", err) + } + + newVersion := currentVer + 1 + + meta := m.GetMeta() + meta.SetVersion(uint8(newVersion)) + meta.SetUpdatedAt(time.Now()) + meta.SetUpdatedBy("qube") + + versionedKey := genVersionedKey(m.GetID(), newVersion) + latestKey := genLatestKey(m.GetID()) + + data, err := json.Marshal(m) if err != nil { - return fmt.Errorf("error marshalling manifest: %v", err) + return fmt.Errorf("marshaling error: %v", err) } - if err = txn.Set(genManifestKey(m.GetID()), data); err != nil { + if err = txn.Set(versionedKey, data); err != nil { return err } - if err = s.index.Index(m.GetID(), m.Index()); err != nil { - ui.Errorf("Failed to index manifest %s: %v", m.GetID(), err) + if err = txn.Set(latestKey, versionedKey); err != nil { + return err + } + + if err = s.indexCurrentManifest(m, string(versionedKey), currentVer); err != nil { return err } - } + if newVersion > MaxManifestVersion { + if err = s.cleanupOldVersions(txn, m.GetID(), MaxManifestVersion); err != nil { + return err + } + } + } return nil }) - - return err } -func (s *Storage) LoadManifests(ids ...string) ([]manifests.Manifest, error) { +func (s *Storage) Load(opts LoadOptions) ([]manifests.Manifest, error) { var results []manifests.Manifest - var rErr error - err := instance.db.View(func(txn *badger.Txn) error { - var item *badger.Item - var err error + if opts.Hash != "" { + m, err := s.loadByHash(opts.Hash) + if err != nil { + return nil, err + } + return []manifests.Manifest{m}, nil + } - for _, id := range ids { - item, err = txn.Get(genManifestKey(id)) - if errors.Is(err, badger.ErrKeyNotFound) { - rErr = errors.Join(rErr, fmt.Errorf("manifest %s not found", id)) - continue - } else if err != nil { - rErr = errors.Join(rErr, err) - continue + if len(opts.Versions) > 0 { + for id, version := range opts.Versions { + m, err := s.loadVersion(id, version) + if err != nil { + return nil, fmt.Errorf("failed to load %s@v%d: %w", id, version, err) } + results = append(results, m) + } + return results, nil + } - var man manifests.Manifest + if len(opts.IDs) > 0 { + return s.loadBulk(opts) + } - if err = item.Value(func(data []byte) error { - if man, err = parsing.ParseManifestAsJSON(data); err == nil { - results = append(results, man) - } + if opts.All || opts.Latest { + return s.loadLatest() + } - return err - }); err != nil { - rErr = errors.Join(rErr, err) - } + return nil, fmt.Errorf("no valid load options specified") +} + +func (s *Storage) Search(query Query) ([]manifests.Manifest, error) { + searchResult, err := query.Execute(s.index) + if err != nil { + return nil, fmt.Errorf("failed to search manifests: %w", err) + } + + return s.parseSearchResults(searchResult) +} + +func (s *Storage) Rollback(id string, targetVersion int) error { + return instance.db.Update(func(txn *badger.Txn) error { + key := genVersionedKey(id, targetVersion) + item, err := txn.Get(key) + if err != nil { + return fmt.Errorf("old version for manifest %s not found", id) } - return nil - }) + val, _ := item.ValueCopy(nil) + newVersion, err := s.getCurrentVersion(txn, id) + if err != nil { + return err + } + + newVersion += 1 + newKey := genVersionedKey(id, newVersion) + + if err = txn.Set(newKey, val); err != nil && errors.Is(err, badger.ErrKeyNotFound) { + return fmt.Errorf("old version for manifest %s not found", id) + } else if err != nil { + return fmt.Errorf("rollback failed: %w", err) + } - return results, errors.Join(rErr, err) + latestKey := genLatestKey(id) + return txn.Set(latestKey, newKey) + }) } -func (s *Storage) LoadManifest(id string) (manifests.Manifest, error) { - var result manifests.Manifest - var rErr error +func (s *Storage) CleanupOldVersions(id string, keep int) error { + return instance.db.Update(func(txn *badger.Txn) error { + versions, err := s.getAllVersions(txn, id) + if err != nil { + return err + } - err := instance.db.View(func(txn *badger.Txn) error { - item, err := txn.Get(genManifestKey(id)) - if errors.Is(err, badger.ErrKeyNotFound) { - rErr = errors.Join(rErr, fmt.Errorf("manifest %s not found", id)) + if len(versions) <= keep { return nil - } else if err != nil { - rErr = errors.Join(rErr, err) } - if err = item.Value(func(data []byte) error { - if result, err = parsing.ParseManifestAsJSON(data); err != nil { - return err + sort.Ints(versions) + toDelete := versions[:len(versions)-keep] + + for _, ver := range toDelete { + key := genVersionedKey(id, ver) + + if err = txn.Delete(key); err != nil { + return fmt.Errorf("error deleting old versions: %w", err) } - return nil - }); err != nil { - rErr = errors.Join(rErr, err) + if err = s.index.Delete(string(key)); err != nil { + return fmt.Errorf("failed to delete from index: %v", err) + } + } + + latestVer := versions[len(versions)-1] + if latestVer > MaxManifestVersion { + return s.resetVersions(txn, id) } return nil }) - - return result, errors.Join(rErr, err) } -func (s *Storage) FindManifestsByKind(kind string) ([]manifests.Manifest, error) { - query := bleve.NewTermQuery(kind) - query.SetField(index.Kind) +func (s *Storage) loadByHash(hash string) (manifests.Manifest, error) { + var manifest manifests.Manifest - searchRequest := bleve.NewSearchRequest(query) + hashQuery := bleve.NewTermQuery(hash) + hashQuery.SetField(index.MetaHash) - searchResults, err := s.index.Search(searchRequest) + searchResult, err := s.index.Search(bleve.NewSearchRequest(hashQuery)) if err != nil { - return nil, fmt.Errorf("error searching manifests: %v", err) + return nil, fmt.Errorf("bleve search failed: %w", err) } - return s.parseSearResults(searchResults) -} + if searchResult.Total == 0 { + return nil, fmt.Errorf("manifest with hash %q not found", hash) + } -func (s *Storage) FindManifestsByVersion(lowVersion, heightVersion uint8) ([]manifests.Manifest, error) { - low, height := float64(lowVersion), float64(heightVersion) + hit := searchResult.Hits[0] - query := bleve.NewNumericRangeQuery(&low, &height) - query.SetField(index.Version) + err = s.db.View(func(txn *badger.Txn) error { + var latestItem *badger.Item - searchRequest := bleve.NewSearchRequest(query) + latestKey := genLatestKey(extractBaseID(hit.ID)) + latestItem, err = txn.Get(latestKey) + if err != nil { + return fmt.Errorf("failed to get latest version: %w", err) + } + var versionedKey []byte + versionedKey, err = latestItem.ValueCopy(nil) + if err != nil { + return fmt.Errorf("failed to get versioned key: %w", err) + } + + var manifestItem *badger.Item + manifestItem, err = txn.Get(versionedKey) + if err != nil { + return fmt.Errorf("failed to get manifest data: %w", err) + } - searchResults, err := s.index.Search(searchRequest) + return manifestItem.Value(func(data []byte) error { + var parseErr error + manifest, parseErr = parsing.ParseManifest(parsing.JSONMethod, data) + return parseErr + }) + }) if err != nil { - return nil, fmt.Errorf("error searching manifests: %v", err) + return nil, fmt.Errorf("storage error: %w", err) + } + + if manifest.GetMeta().GetHash() != hash { + return nil, fmt.Errorf("hash mismatch after load") } - return s.parseSearResults(searchResults) + return manifest, nil } -func (s *Storage) FindManifestByName(name string) (manifests.Manifest, error) { - query := bleve.NewMatchPhraseQuery(name) - query.SetField(index.Name) +func (s *Storage) loadVersion(id string, version int) (manifests.Manifest, error) { + var m manifests.Manifest + versionedKey := genVersionedKey(id, version) - searchRequest := bleve.NewSearchRequest(query) - searchRequest.Size = 1 + err := s.db.View(func(txn *badger.Txn) error { + item, err := txn.Get(versionedKey) + if err != nil { + return err + } + return item.Value(func(data []byte) error { + m, err = parsing.ParseManifestAsJSON(data) + return err + }) + }) - searchResults, err := s.index.Search(searchRequest) - if err != nil { - return nil, fmt.Errorf("search failed: %w", err) - } + return m, err +} - results, err := s.parseSearResults(searchResults) - if err != nil { - return nil, err - } +func (s *Storage) loadLatest() ([]manifests.Manifest, error) { + var results []manifests.Manifest - if len(results) == 0 { - return nil, fmt.Errorf("no matching manifest found") - } else { - return results[0], nil - } -} + err := s.db.View(func(txn *badger.Txn) error { + opts := badger.DefaultIteratorOptions + opts.Prefix = []byte(manifestsLatestKey) + it := txn.NewIterator(opts) + defer it.Close() + + for it.Rewind(); it.Valid(); it.Next() { + item := it.Item() -func (s *Storage) FindManifestsByNameWildcard(namePattern string) ([]manifests.Manifest, error) { - query := bleve.NewWildcardQuery(fmt.Sprintf("*%s*", namePattern)) - query.SetField(index.Name) + versionedKey, err := item.ValueCopy(nil) + if err != nil { + return fmt.Errorf("failed to get versioned key for %s: %w", item.Key(), err) + } - searchRequest := bleve.NewSearchRequest(query) + versionedItem, err := txn.Get(versionedKey) + if err != nil { + return fmt.Errorf("failed to get manifest at %s: %w", versionedKey, err) + } - searchResults, err := s.index.Search(searchRequest) + var m manifests.Manifest + err = versionedItem.Value(func(data []byte) error { + m, err = parsing.ParseManifestAsJSON(data) + if err != nil { + return fmt.Errorf("failed to parse manifest %s: %w", versionedKey, err) + } + results = append(results, m) + return nil + }) + if err != nil { + return err + } + } + return nil + }) if err != nil { - return nil, fmt.Errorf("search failed: %w", err) + return nil, fmt.Errorf("failed to load results: %w", err) } - return s.parseSearResults(searchResults) + return results, nil } -func (s *Storage) FindManifestsByNamespace(namespace string) ([]manifests.Manifest, error) { - query := bleve.NewTermQuery(namespace) - query.SetField(index.Namespace) +func (s *Storage) loadBulk(opts LoadOptions) ([]manifests.Manifest, error) { + var ( + results []manifests.Manifest + errs error + ) - searchRequest := bleve.NewSearchRequest(query) + err := instance.db.View(func(txn *badger.Txn) error { + for _, id := range opts.IDs { + item, err := txn.Get(genLatestKey(id)) + if err != nil { + if errors.Is(err, badger.ErrKeyNotFound) { + errs = errors.Join(errs, fmt.Errorf("manifest %q not found", id)) + continue + } + errs = errors.Join(errs, fmt.Errorf("failed to get latest pointer for %q: %w", id, err)) + continue + } - searchResults, err := s.index.Search(searchRequest) - if err != nil { - return nil, fmt.Errorf("search failed: %w", err) - } + var latestKey []byte + if err = item.Value(func(val []byte) error { + latestKey = append([]byte{}, val...) + return nil + }); err != nil { + errs = errors.Join(errs, fmt.Errorf("failed to read latest key for %q: %w", id, err)) + continue + } - return s.parseSearResults(searchResults) -} + item, err = txn.Get(latestKey) + if err != nil { + errs = errors.Join(errs, fmt.Errorf("failed to get manifest data for %q: %w", id, err)) + continue + } -func (s *Storage) FindManifestByDependencies(dependencies []string, requireAll bool) ([]manifests.Manifest, error) { - var query bleceQuery.Query + var manifest manifests.Manifest + if err = item.Value(func(data []byte) error { + manifest, err = parsing.ParseManifestAsJSON(data) + return err + }); err != nil { + errs = errors.Join(errs, fmt.Errorf("parsing failed for %q: %w", id, err)) + continue + } - if requireAll { - q := bleve.NewConjunctionQuery() - for _, dep := range dependencies { - termQuery := bleve.NewTermQuery(dep) - termQuery.SetField(index.DependsOn) - q.AddQuery(termQuery) + results = append(results, manifest) } - query = q - } else { - q := bleve.NewDisjunctionQuery() - for _, dep := range dependencies { - termQuery := bleve.NewTermQuery(dep) - termQuery.SetField(index.DependsOn) - q.AddQuery(termQuery) + + if len(results) == 0 && errs != nil { + return fmt.Errorf("no manifests loaded: %w", errs) } - query = q + return nil + }) + if err != nil { + return results, fmt.Errorf("transaction failed: %w", errors.Join(err, errs)) } - searchRequest := bleve.NewSearchRequest(query) + return results, errs +} - searchResults, err := s.index.Search(searchRequest) - if err != nil { - return nil, fmt.Errorf("search failed: %w", err) +func (s *Storage) parseSearchResults(searchResults *bleve.SearchResult) ([]manifests.Manifest, error) { + if searchResults == nil || searchResults.Total == 0 { + return nil, nil } - return s.parseSearResults(searchResults) -} + var ( + manifestsList = make([]manifests.Manifest, 0, len(searchResults.Hits)) + errorsList []error + ) -func (s *Storage) FindManifestByHash(hash string) (manifests.Manifest, error) { - query := bleve.NewTermQuery(hash) - query.SetField(index.MetaHash) + err := s.db.View(func(txn *badger.Txn) error { + for _, hit := range searchResults.Hits { + latestKey := genLatestKey(extractBaseID(hit.ID)) + item, err := txn.Get(latestKey) + if err != nil { + errorsList = append(errorsList, fmt.Errorf("latest key for %q: %w", hit.ID, err)) + continue + } - searchRequest := bleve.NewSearchRequest(query) - searchRequest.Size = 1 + versionedKey, err := item.ValueCopy(nil) + if err != nil { + errorsList = append(errorsList, fmt.Errorf("value copy for %q: %w", hit.ID, err)) + continue + } - searchResults, err := s.index.Search(searchRequest) - if err != nil { - return nil, fmt.Errorf("search failed: %w", err) - } + manifestItem, err := txn.Get(versionedKey) + if err != nil { + errorsList = append(errorsList, fmt.Errorf("manifest %q: %w", string(versionedKey), err)) + continue + } - results, err := s.parseSearResults(searchResults) + var m manifests.Manifest + err = manifestItem.Value(func(data []byte) error { + m, err = parsing.ParseManifest(parsing.JSONMethod, data) + if err != nil { + return err + } + manifestsList = append(manifestsList, m) + return nil + }) + if err != nil { + errorsList = append(errorsList, fmt.Errorf("parse manifest %q: %w", string(versionedKey), err)) + } + } + return nil + }) if err != nil { - return nil, err + return nil, fmt.Errorf("transaction failed: %w", err) } - if len(results) == 0 { - return nil, fmt.Errorf("no matching manifest found") - } else { - return results[0], nil - } + return manifestsList, errors.Join(errorsList...) } -func (s *Storage) FindManifestsByCreatedAtRange(start, end time.Time) ([]manifests.Manifest, error) { - query := bleve.NewDateRangeQuery(start, end) - query.SetField(index.MetaCreatedAt) - - searchRequest := bleve.NewSearchRequest(query) +func (s *Storage) getCurrentVersion(txn *badger.Txn, id string) (int, error) { + item, err := txn.Get(genLatestKey(id)) + if errors.Is(err, badger.ErrKeyNotFound) { + return 0, nil + } + if err != nil { + return 0, err + } - searchResults, err := s.index.Search(searchRequest) + val, err := item.ValueCopy(nil) if err != nil { - return nil, fmt.Errorf("error searching manifests: %v", err) + return 0, err } - return s.parseSearResults(searchResults) + valStr := string(val) + parts := strings.Split(valStr, "@v") + if len(parts) != 2 { + return 0, fmt.Errorf("invalid versioned key format") + } + + return strconv.Atoi(parts[1]) } -func (s *Storage) FindManifestsByCreatedBy(createdBy string) ([]manifests.Manifest, error) { - query := bleve.NewTermQuery(createdBy) - query.SetField(index.MetaCreatedBy) +func (s *Storage) indexCurrentManifest(m manifests.Manifest, newKey string, oldVersion int) error { + if oldVersion > 0 { + oldKey := genVersionedKey(m.GetID(), oldVersion) + if err := s.index.Delete(string(oldKey)); err != nil { + return fmt.Errorf("could not delete manifest: %s old version %v: %v", m.GetID(), oldVersion, err) + } + } - searchRequest := bleve.NewSearchRequest(query) + m.GetMeta().SetIsCurrent(true) - searchResults, err := s.index.Search(searchRequest) - if err != nil { - return nil, fmt.Errorf("search failed: %w", err) + if err := s.index.Index(newKey, m.Index()); err != nil { + return fmt.Errorf("could not index manifest: %s old version %v: %v", m.GetID(), oldVersion, err) } - return s.parseSearResults(searchResults) + return nil } -func (s *Storage) FindManifestsByUpdatedAtRange(start, end time.Time) ([]manifests.Manifest, error) { - query := bleve.NewDateRangeQuery(start, end) - query.SetField(index.MetaUpdatedAt) - - searchRequest := bleve.NewSearchRequest(query) - - searchResults, err := s.index.Search(searchRequest) +func (s *Storage) cleanupOldVersions(txn *badger.Txn, id string, keepVersions int) error { + versions, err := s.getAllVersions(txn, id) if err != nil { - return nil, fmt.Errorf("error searching manifests: %v", err) + return err } - return s.parseSearResults(searchResults) -} + sort.Ints(versions) -func (s *Storage) FindManifestsByUpdatedBy(updatedBy string) ([]manifests.Manifest, error) { - query := bleve.NewTermQuery(updatedBy) - query.SetField(index.MetaUpdatedBy) + for i := 0; i < len(versions)-keepVersions; i++ { + key := genVersionedKey(id, versions[i]) - searchRequest := bleve.NewSearchRequest(query) + if err = txn.Delete(key); err != nil { + return fmt.Errorf("could not delete manifest: %s old version %v: %v", id, versions[i], err) + } - searchResults, err := s.index.Search(searchRequest) - if err != nil { - return nil, fmt.Errorf("search failed: %w", err) + if err = s.index.Delete(string(key)); err != nil { + return fmt.Errorf("could not delete manifest: %s old version %v: %v", id, versions[i], err) + } } - return s.parseSearResults(searchResults) + if versions[len(versions)-1] > MaxManifestVersion { + return s.resetVersions(txn, id) + } + + return nil } -func (s *Storage) FindManifestsByUsedBy(usedBy string) ([]manifests.Manifest, error) { - query := bleve.NewTermQuery(usedBy) - query.SetField(index.MetaUsedBy) +func (s *Storage) resetVersions(txn *badger.Txn, id string) error { + latestKey := genLatestKey(id) + item, err := txn.Get(latestKey) + if err != nil { + return fmt.Errorf("failed to get latest version: %w", err) + } - searchRequest := bleve.NewSearchRequest(query) + latestVal, err := item.ValueCopy(nil) + if err != nil { + return fmt.Errorf("failed to read latest value: %w", err) + } - searchResults, err := s.index.Search(searchRequest) + item, err = txn.Get(latestVal) if err != nil { - return nil, fmt.Errorf("search failed: %w", err) + return fmt.Errorf("failed to get manifest data: %w", err) } - return s.parseSearResults(searchResults) -} + manifestData, err := item.ValueCopy(nil) + if err != nil { + return fmt.Errorf("failed to copy manifest data: %w", err) + } -func (s *Storage) FindManifestsByLastAppliedRange(start, end time.Time) ([]manifests.Manifest, error) { - query := bleve.NewDateRangeQuery(start, end) - query.SetField(index.MetaLastApplied) + manifest, err := parsing.ParseManifest(parsing.JSONMethod, manifestData) + if err != nil { + return fmt.Errorf("parsing failed: %w", err) + } - searchRequest := bleve.NewSearchRequest(query) + meta := manifest.GetMeta() + meta.SetVersion(1) + meta.SetUpdatedAt(time.Now()) + meta.SetUpdatedBy("qube") + meta.SetIsCurrent(true) - searchResults, err := s.index.Search(searchRequest) + newData, err := json.Marshal(manifest) if err != nil { - return nil, fmt.Errorf("error searching manifests: %v", err) + return fmt.Errorf("marshaling failed: %w", err) } - return s.parseSearResults(searchResults) -} + newKey := genVersionedKey(id, 1) + if err = txn.Set(newKey, newData); err != nil { + return fmt.Errorf("failed to save new version: %w", err) + } -func (s *Storage) parseSearResults(searchResults *bleve.SearchResult) ([]manifests.Manifest, error) { - var results []manifests.Manifest - var rErr error + if err = txn.Set(latestKey, newKey); err != nil { + return fmt.Errorf("failed to update latest pointer: %w", err) + } - if searchResults.Total > 0 { - for _, hit := range searchResults.Hits { - var man manifests.Manifest + if err = s.cleanupIndexForReset(id, newKey); err != nil { + return fmt.Errorf("index cleanup failed: %w", err) + } - err := s.db.View(func(txn *badger.Txn) error { - var item *badger.Item - var err error + return s.index.Index(string(newKey), manifest.Index()) +} - item, err = txn.Get(genManifestKey(hit.ID)) - if err != nil && errors.Is(err, badger.ErrKeyNotFound) { - rErr = errors.Join(rErr, fmt.Errorf("could not find manifest by ID: %v", hit.ID)) - } else if err != nil { - rErr = errors.Join(rErr, fmt.Errorf("failed to load manifest by ID: %v", hit.ID)) - } +func (s *Storage) cleanupIndexForReset(id string, newKey []byte) error { + query := bleve.NewTermQuery(id) + searchReq := bleve.NewSearchRequest(query) + searchReq.Fields = []string{"*"} - return item.Value(func(val []byte) error { - man, err = parsing.ParseManifest(parsing.JSONMethod, val) - if err != nil { - return err - } - return nil - }) - }) - if err != nil { - rErr = errors.Join(rErr, err) - } + searchResults, err := s.index.Search(searchReq) + if err != nil { + return err + } - results = append(results, man) + var delErrors error + for _, hit := range searchResults.Hits { + if hit.ID != string(newKey) { + if err := s.index.Delete(hit.ID); err != nil { + delErrors = errors.Join(delErrors, + fmt.Errorf("failed to delete %s: %w", hit.ID, err)) + } } } + return delErrors +} - return results, rErr +func (s *Storage) getAllVersions(txn *badger.Txn, id string) ([]int, error) { + var versions []int + it := txn.NewIterator(badger.DefaultIteratorOptions) + defer it.Close() + + prefix := []byte(id + "@v") + for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() { + key := string(it.Item().Key()) + parts := strings.Split(key, "@v") + if len(parts) != 2 { + continue + } + ver, err := strconv.Atoi(parts[1]) + if err != nil { + continue + } + versions = append(versions, ver) + } + return versions, nil } diff --git a/internal/core/store/helpers.go b/internal/core/store/helpers.go index f7cf4b6..df172e1 100644 --- a/internal/core/store/helpers.go +++ b/internal/core/store/helpers.go @@ -1,5 +1,25 @@ package store -func genManifestKey(id string) []byte { - return []byte(id) +import ( + "fmt" + "strings" +) + +const ( + manifestsLatestKey = "latest/" +) + +func genLatestKey(id string) []byte { + return []byte(fmt.Sprintf("%s%s", manifestsLatestKey, id)) +} + +func genVersionedKey(id string, version int) []byte { + return []byte(fmt.Sprintf("%s@v%d", id, version)) +} + +func extractBaseID(versionedKey string) string { + if at := strings.LastIndex(versionedKey, "@"); at != -1 { + return versionedKey[:at] + } + return versionedKey } diff --git a/internal/core/store/independ.go b/internal/core/store/independ.go index 946485b..d941fec 100644 --- a/internal/core/store/independ.go +++ b/internal/core/store/independ.go @@ -1,13 +1,15 @@ package store import ( + "errors" "sync" - "time" "github.com/apiqube/cli/internal/core/manifests" "github.com/apiqube/cli/ui" ) +var errStoreNotInitialized = errors.New("store not initialized") + var ( instance *Storage once sync.Once @@ -47,138 +49,42 @@ func IsEnabled() bool { return instance != nil && enabled } -func SaveManifests(mans ...manifests.Manifest) error { - if !isEnabled() { - return nil - } - - return instance.SaveManifests(mans...) -} - -func LoadManifests(ids ...string) ([]manifests.Manifest, error) { - if !isEnabled() { - return nil, nil - } - - return instance.LoadManifests() -} - -func LoadManifest(id string) (manifests.Manifest, error) { - if !isEnabled() { - return nil, nil - } - - return instance.LoadManifest(id) -} - -func FindManifestsByKind(kind string) ([]manifests.Manifest, error) { - if !isEnabled() { - return nil, nil - } - - return instance.FindManifestsByKind(kind) -} - -func FindManifestsByVersion(lowVersion, heightVersion uint8) ([]manifests.Manifest, error) { - if !isEnabled() { - return nil, nil - } - - return instance.FindManifestsByVersion(lowVersion, heightVersion) -} - -func FindManifestByName(name string) (manifests.Manifest, error) { - if !isEnabled() { - return nil, nil - } - - return instance.FindManifestByName(name) -} - -func FindManifestsByNameWildcard(namePattern string) ([]manifests.Manifest, error) { - if !isEnabled() { - return nil, nil - } - - return instance.FindManifestsByNameWildcard(namePattern) -} - -func FindManifestsByNamespace(namespace string) ([]manifests.Manifest, error) { - if !isEnabled() { - return nil, nil - } - - return instance.FindManifestsByNamespace(namespace) -} - -func FindManifestByDependencies(dependencies []string, requireAll bool) ([]manifests.Manifest, error) { - if !isEnabled() { - return nil, nil - } - - return instance.FindManifestByDependencies(dependencies, requireAll) -} - -func FindManifestByHash(hash string) (manifests.Manifest, error) { - if !isEnabled() { - return nil, nil - } - - return instance.FindManifestByHash(hash) -} - -func FindManifestsByCreatedAtRange(start, end time.Time) ([]manifests.Manifest, error) { - if !isEnabled() { - return nil, nil - } - - return instance.FindManifestsByCreatedAtRange(start, end) -} - -func FindManifestsByCreatedBy(createdBy string) ([]manifests.Manifest, error) { - if !isEnabled() { - return nil, nil - } - - return instance.FindManifestsByCreatedBy(createdBy) -} - -func FindManifestsByUpdatedAtRange(start, end time.Time) ([]manifests.Manifest, error) { - if !isEnabled() { - return nil, nil +func Save(mans ...manifests.Manifest) error { + if !IsEnabled() { + return errStoreNotInitialized } - return instance.FindManifestsByUpdatedAtRange(start, end) + return instance.Save(mans...) } -func FindManifestsByUpdatedBy(updatedBy string) ([]manifests.Manifest, error) { - if !isEnabled() { - return nil, nil +func Load(opts LoadOptions) ([]manifests.Manifest, error) { + if !IsEnabled() { + return nil, errStoreNotInitialized } - return instance.FindManifestsByUpdatedBy(updatedBy) + return instance.Load(opts) } -func FindManifestsByUsedBy(usedBy string) ([]manifests.Manifest, error) { - if !isEnabled() { - return nil, nil +func Search(query Query) ([]manifests.Manifest, error) { + if !IsEnabled() { + return nil, errStoreNotInitialized } - return instance.FindManifestsByUsedBy(usedBy) + return instance.Search(query) } -func FindManifestsByLastAppliedRange(start, end time.Time) ([]manifests.Manifest, error) { - if !isEnabled() { - return nil, nil +func Rollback(id string, targetVersion int) error { + if !IsEnabled() { + return errStoreNotInitialized } - return instance.FindManifestsByLastAppliedRange(start, end) + return instance.Rollback(id, targetVersion) } -func isEnabled() bool { +func CleanupOldVersions(id string, keep int) error { if !IsEnabled() { - ui.Errorf("Database instance not ready") - return false + return errStoreNotInitialized } - return true + + return instance.CleanupOldVersions(id, keep) } diff --git a/internal/core/store/index.go b/internal/core/store/index.go index 065a55f..7792215 100644 --- a/internal/core/store/index.go +++ b/internal/core/store/index.go @@ -7,36 +7,62 @@ import ( ) func buildBleveMapping() *mapping.IndexMappingImpl { + indexMapping := bleve.NewIndexMapping() + indexMapping.DefaultAnalyzer = "standard" + manifestMapping := bleve.NewDocumentMapping() - // Main + idMapping := bleve.NewTextFieldMapping() + idMapping.Analyzer = "keyword" + idMapping.Store = true + manifestMapping.AddFieldMappingsAt(index.ID, idMapping) + manifestMapping.AddFieldMappingsAt(index.Version, bleve.NewNumericFieldMapping()) + manifestMapping.AddFieldMappingsAt(index.MetaVersion, bleve.NewNumericFieldMapping()) - kindMapping := bleve.NewTextFieldMapping() - kindMapping.Analyzer = "keyword" - manifestMapping.AddFieldMappingsAt(index.Kind, kindMapping) + exactMatchFields := []string{ + index.Kind, + index.DependsOn, + index.MetaCreatedBy, + index.MetaUpdatedBy, + index.MetaUsedBy, + } + + for _, field := range exactMatchFields { + fm := bleve.NewTextFieldMapping() + fm.Analyzer = "keyword" + manifestMapping.AddFieldMappingsAt(field, fm) + } nameMapping := bleve.NewTextFieldMapping() + nameMapping.Analyzer = "keyword" manifestMapping.AddFieldMappingsAt(index.Name, nameMapping) - dependsMapping := bleve.NewTextFieldMapping() - dependsMapping.Analyzer = "keyword" - manifestMapping.AddFieldMappingsAt(index.DependsOn, dependsMapping) - namespaceMapping := bleve.NewTextFieldMapping() namespaceMapping.Analyzer = "keyword" manifestMapping.AddFieldMappingsAt(index.Namespace, namespaceMapping) - // Meta - manifestMapping.AddFieldMappingsAt(index.MetaHash, bleve.NewTextFieldMapping()) - manifestMapping.AddFieldMappingsAt(index.MetaCreatedAt, bleve.NewDateTimeFieldMapping()) - manifestMapping.AddFieldMappingsAt(index.MetaCreatedBy, bleve.NewTextFieldMapping()) - manifestMapping.AddFieldMappingsAt(index.MetaUpdatedAt, bleve.NewDateTimeFieldMapping()) - manifestMapping.AddFieldMappingsAt(index.MetaUpdatedBy, bleve.NewTextFieldMapping()) - manifestMapping.AddFieldMappingsAt(index.MetaUsedBy, bleve.NewTextFieldMapping()) - manifestMapping.AddFieldMappingsAt(index.MetaLastApplied, bleve.NewDateTimeFieldMapping()) + hashMapping := bleve.NewTextFieldMapping() + hashMapping.Analyzer = "keyword" + hashMapping.Store = true + manifestMapping.AddFieldMappingsAt(index.MetaHash, hashMapping) + + dateTimeFieldMapping := bleve.NewTextFieldMapping() + dateTimeFieldMapping.Analyzer = "keyword" + dateTimeFieldMapping.Store = true + dateTimeFieldMapping.IncludeTermVectors = false + dateTimeFieldMapping.IncludeInAll = false + + dateFields := []string{ + index.MetaCreatedAt, + index.MetaUpdatedAt, + index.MetaLastApplied, + } + + for _, field := range dateFields { + manifestMapping.AddFieldMappingsAt(field, dateTimeFieldMapping) + } - indexMapping := bleve.NewIndexMapping() indexMapping.DefaultMapping = manifestMapping return indexMapping diff --git a/internal/core/store/query.go b/internal/core/store/query.go new file mode 100644 index 0000000..84113c3 --- /dev/null +++ b/internal/core/store/query.go @@ -0,0 +1,183 @@ +package store + +import ( + "time" + + "github.com/apiqube/cli/internal/core/manifests/index" + "github.com/blevesearch/bleve/v2" + "github.com/blevesearch/bleve/v2/search/query" +) + +type Query interface { + Execute(index bleve.Index) (*bleve.SearchResult, error) + WithAll() Query + WithExactName(name string) Query + WithWildcardName(pattern string) Query + WithRegexName(regex string) Query + WithKind(kind string) Query + WithNamespace(namespace string) Query + WithVersion(version int) Query + WithCreatedBy(by string) Query + WithUsedBy(by string) Query + WithHashPrefix(prefix string) Query + WithDependencies(deps []string) Query + WithAllDependencies(deps []string) Query + WithCreatedAfter(t time.Time) Query + WithCreatedBefore(t time.Time) Query + WithUpdatedAfter(t time.Time) Query + WithUpdatedBefore(t time.Time) Query + WithLastApplied(t time.Time) Query +} + +type manifestQuery struct { + bleveQuery query.Query + limit int +} + +func NewQuery() Query { + return &manifestQuery{ + bleveQuery: bleve.NewMatchAllQuery(), + } +} + +func (q *manifestQuery) Execute(index bleve.Index) (*bleve.SearchResult, error) { + searchRequest := bleve.NewSearchRequest(q.bleveQuery) + if q.limit > 0 { + searchRequest.Size = q.limit + } + + return index.Search(searchRequest) +} + +func (q *manifestQuery) WithAll() Query { + q.bleveQuery = bleve.NewMatchAllQuery() + return q +} + +func (q *manifestQuery) WithExactName(name string) Query { + termQuery := bleve.NewTermQuery(name) + termQuery.SetField(index.Name) + q.addQuery(termQuery) + return q +} + +func (q *manifestQuery) WithWildcardName(pattern string) Query { + wildcardQuery := bleve.NewWildcardQuery(pattern) + wildcardQuery.SetField(index.Name) + q.addQuery(wildcardQuery) + return q +} + +func (q *manifestQuery) WithRegexName(regex string) Query { + regexpQuery := bleve.NewRegexpQuery(regex) + regexpQuery.SetField(index.Name) + q.addQuery(regexpQuery) + return q +} + +func (q *manifestQuery) WithKind(kind string) Query { + termQuery := bleve.NewTermQuery(kind) + termQuery.SetField(index.Kind) + q.addQuery(termQuery) + return q +} + +func (q *manifestQuery) WithNamespace(namespace string) Query { + termQuery := bleve.NewTermQuery(namespace) + termQuery.SetField(index.Namespace) + q.addQuery(termQuery) + return q +} + +func (q *manifestQuery) WithVersion(version int) Query { + val := float64(version) + numericQuery := bleve.NewNumericRangeQuery(&val, nil) + numericQuery.SetField(index.MetaVersion) + return q +} + +func (q *manifestQuery) WithCreatedBy(by string) Query { + termQuery := bleve.NewTermQuery(by) + termQuery.SetField(index.MetaCreatedBy) + q.addQuery(termQuery) + return q +} + +func (q *manifestQuery) WithUsedBy(by string) Query { + termQuery := bleve.NewTermQuery(by) + termQuery.SetField(index.MetaUsedBy) + q.addQuery(termQuery) + return q +} + +func (q *manifestQuery) WithHashPrefix(prefix string) Query { + prefixQuery := bleve.NewPrefixQuery(prefix) + prefixQuery.SetField(index.MetaHash) + q.addQuery(prefixQuery) + return q +} + +func (q *manifestQuery) WithDependencies(deps []string) Query { + disjunctionQuery := bleve.NewDisjunctionQuery() + for _, dep := range deps { + termQuery := bleve.NewTermQuery(dep) + termQuery.SetField(index.DependsOn) + disjunctionQuery.AddQuery(termQuery) + } + q.addQuery(disjunctionQuery) + return q +} + +func (q *manifestQuery) WithAllDependencies(deps []string) Query { + conjunctionQuery := bleve.NewConjunctionQuery() + for _, dep := range deps { + termQuery := bleve.NewTermQuery(dep) + termQuery.SetField(index.DependsOn) + conjunctionQuery.AddQuery(termQuery) + } + q.addQuery(conjunctionQuery) + return q +} + +func (q *manifestQuery) WithCreatedAfter(t time.Time) Query { + dateQuery := bleve.NewTermRangeQuery(t.Format(time.RFC3339Nano), "") + dateQuery.SetField(index.MetaCreatedAt) + q.addQuery(dateQuery) + return q +} + +func (q *manifestQuery) WithCreatedBefore(t time.Time) Query { + dateQuery := bleve.NewTermRangeQuery("", t.Format(time.RFC3339Nano)) + dateQuery.SetField(index.MetaCreatedAt) + q.addQuery(dateQuery) + return q +} + +func (q *manifestQuery) WithUpdatedAfter(t time.Time) Query { + dateQuery := bleve.NewTermRangeQuery(t.Format(time.RFC3339Nano), "") + dateQuery.SetField(index.MetaUpdatedAt) + q.addQuery(dateQuery) + return q +} + +func (q *manifestQuery) WithUpdatedBefore(t time.Time) Query { + dateQuery := bleve.NewTermRangeQuery("", t.Format(time.RFC3339Nano)) + dateQuery.SetField(index.MetaUpdatedAt) + q.addQuery(dateQuery) + return q +} + +func (q *manifestQuery) WithLastApplied(t time.Time) Query { + dateQuery := bleve.NewTermRangeQuery("", t.Format(time.RFC3339Nano)) + dateQuery.SetField(index.MetaLastApplied) + q.addQuery(dateQuery) + return q +} + +func (q *manifestQuery) addQuery(newQuery query.Query) { + if _, ok := q.bleveQuery.(*query.MatchAllQuery); ok { + q.bleveQuery = newQuery + } else { + q.bleveQuery = bleve.NewConjunctionQuery(q.bleveQuery, newQuery) + } +} diff --git a/ui/styles.go b/ui/styles.go index e475c64..4893a94 100644 --- a/ui/styles.go +++ b/ui/styles.go @@ -18,8 +18,8 @@ var ( Foreground(lipgloss.Color("#5fd700")) errorStyle = lipgloss.NewStyle(). - Foreground(lipgloss.Color("#d70000")). - Bold(true) + Foreground(lipgloss.Color("#FF0000")). + Bold(false) warningStyle = lipgloss.NewStyle(). Foreground(lipgloss.Color("#ff8700")). diff --git a/ui/ui.go b/ui/ui.go index 2656d7e..52c32a4 100644 --- a/ui/ui.go +++ b/ui/ui.go @@ -2,6 +2,7 @@ package ui import ( "fmt" + "os" "sync" "time" @@ -74,8 +75,8 @@ func Init() { go func() { ui.program = tea.NewProgram( model, - tea.WithoutSignalHandler(), tea.WithInput(nil), + tea.WithOutput(os.Stderr), ) close(ui.ready) if _, err := ui.program.Run(); err != nil {