Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ __azurite*
go.work.sum

# Taco specific binaries
taco

statesman
terraform-provider-opentaco
opentacosvc
Expand All @@ -35,3 +35,6 @@ bin/
*.swp
*.swo
*~

#data
data/
25 changes: 25 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
@@ -1,3 +1,28 @@
module github.com/diggerhq/digger

go 1.24.0

require (
filippo.io/edwards25519 v1.1.0 // indirect
github.com/go-sql-driver/mysql v1.8.1 // indirect
github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect
github.com/golang-sql/sqlexp v0.1.0 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/pgx/v5 v5.6.0 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/kelseyhightower/envconfig v1.4.0 // indirect
github.com/mattn/go-sqlite3 v1.14.22 // indirect
github.com/microsoft/go-mssqldb v1.8.2 // indirect
golang.org/x/crypto v0.31.0 // indirect
golang.org/x/sync v0.10.0 // indirect
golang.org/x/text v0.21.0 // indirect
gorm.io/driver/mysql v1.6.0 // indirect
gorm.io/driver/postgres v1.6.0 // indirect
gorm.io/driver/sqlite v1.6.0 // indirect
gorm.io/driver/sqlserver v1.6.1 // indirect
gorm.io/gorm v1.31.0 // indirect
)
210 changes: 210 additions & 0 deletions go.sum

Large diffs are not rendered by default.

21 changes: 19 additions & 2 deletions taco/cmd/statesman/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,12 @@ go 1.24

require (
github.com/diggerhq/digger/opentaco/internal v0.0.0
github.com/kelseyhightower/envconfig v1.4.0
github.com/labstack/echo/v4 v4.11.4
)

require (
filippo.io/edwards25519 v1.1.0 // indirect
github.com/aws/aws-sdk-go-v2 v1.38.1 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0 // indirect
github.com/aws/aws-sdk-go-v2/config v1.31.2 // indirect
Expand All @@ -27,16 +29,25 @@ require (
github.com/aws/aws-sdk-go-v2/service/sts v1.38.0 // indirect
github.com/aws/smithy-go v1.22.5 // indirect
github.com/coreos/go-oidc/v3 v3.11.0 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/go-jose/go-jose/v4 v4.0.2 // indirect
github.com/go-sql-driver/mysql v1.8.1 // indirect
github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
github.com/golang-jwt/jwt/v5 v5.3.0 // indirect
github.com/google/go-cmp v0.7.0 // indirect
github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect
github.com/golang-sql/sqlexp v0.1.0 // indirect
github.com/google/jsonapi v1.0.0 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/pgx/v5 v5.6.0 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/labstack/gommon v0.4.2 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-sqlite3 v1.14.22 // indirect
github.com/microsoft/go-mssqldb v1.8.2 // indirect
github.com/mr-tron/base58 v1.2.0 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/stretchr/testify v1.10.0 // indirect
Expand All @@ -45,9 +56,15 @@ require (
golang.org/x/crypto v0.32.0 // indirect
golang.org/x/net v0.34.0 // indirect
golang.org/x/oauth2 v0.24.0 // indirect
golang.org/x/sync v0.10.0 // indirect
golang.org/x/sys v0.29.0 // indirect
golang.org/x/text v0.21.0 // indirect
golang.org/x/time v0.7.0 // indirect
gorm.io/driver/mysql v1.6.0 // indirect
gorm.io/driver/postgres v1.6.0 // indirect
gorm.io/driver/sqlite v1.6.0 // indirect
gorm.io/driver/sqlserver v1.6.1 // indirect
gorm.io/gorm v1.31.0 // indirect
)

replace github.com/diggerhq/digger/opentaco/internal => ../../internal
221 changes: 219 additions & 2 deletions taco/cmd/statesman/go.sum

Large diffs are not rendered by default.

125 changes: 109 additions & 16 deletions taco/cmd/statesman/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,48 +21,133 @@ import (

"github.com/diggerhq/digger/opentaco/internal/analytics"
"github.com/diggerhq/digger/opentaco/internal/api"
"github.com/diggerhq/digger/opentaco/internal/auth"
"github.com/diggerhq/digger/opentaco/internal/query"
"github.com/diggerhq/digger/opentaco/internal/queryfactory"
"github.com/diggerhq/digger/opentaco/internal/storage"
"github.com/diggerhq/digger/opentaco/internal/wiring"
"github.com/kelseyhightower/envconfig"
"github.com/labstack/echo/v4"
"github.com/labstack/echo/v4/middleware"
echomiddleware "github.com/labstack/echo/v4/middleware"
)

func main() {
var (
port = flag.String("port", "8080", "Server port")
authDisable = flag.Bool("auth-disable", false, "Disable auth enforcement (default: false)")
authDisable = flag.Bool("auth-disable", os.Getenv("OPENTACO_AUTH_DISABLE") == "true", "Disable auth enforcement (default: false)")
storageType = flag.String("storage", "s3", "Storage type: s3 or memory (default: s3 with fallback to memory)")
s3Bucket = flag.String("s3-bucket", os.Getenv("OPENTACO_S3_BUCKET"), "S3 bucket for state storage")
s3Prefix = flag.String("s3-prefix", os.Getenv("OPENTACO_S3_PREFIX"), "S3 key prefix (optional)")
s3Region = flag.String("s3-region", os.Getenv("OPENTACO_S3_REGION"), "S3 region (optional; uses AWS defaults if empty)")
)
flag.Parse()

// Load configuration from environment variables into our struct.
var queryCfg query.Config
err := envconfig.Process("taco", &queryCfg) // The prefix "TACO" will be used for all vars.
if err != nil {
log.Fatalf("Failed to process configuration: %v", err)
}

// --- Initialize Stores ---

// Create the database index store using the dedicated factory.
queryStore, err := queryfactory.NewQueryStore(queryCfg)
if err != nil {
log.Fatalf("Failed to initialize query backend: %v", err)
}
defer queryStore.Close()

log.Printf("Query backend initialized: %s (enabled: %v)", queryCfg.Backend, queryStore.IsEnabled())

if queryStore.IsEnabled(){
log.Println("Query backend enabled successfully")
}else{
log.Println("Query backend disabled. You are in no-op mode.")
}


// Initialize storage
var store storage.UnitStore
var blobStore storage.UnitStore
switch *storageType {
case "s3":
if *s3Bucket == "" {
log.Printf("WARNING: S3 storage selected but bucket not provided. Falling back to in-memory storage.")
store = storage.NewMemStore()
blobStore = storage.NewMemStore()
log.Printf("Using in-memory storage")
break
}
s, err := storage.NewS3Store(context.Background(), *s3Bucket, *s3Prefix, *s3Region)
if err != nil {
log.Printf("WARNING: failed to initialize S3 store: %v. Falling back to in-memory storage.", err)
store = storage.NewMemStore()
blobStore = storage.NewMemStore()
log.Printf("Using in-memory storage")
} else {
store = s
blobStore = s
log.Printf("Using S3 storage: bucket=%s prefix=%s region=%s", *s3Bucket, *s3Prefix, *s3Region)
}

}
default:
store = storage.NewMemStore()
blobStore = storage.NewMemStore()
log.Printf("Using in-memory storage")
}


// 3. Create the base OrchestratingStore
orchestratingStore := storage.NewOrchestratingStore(blobStore, queryStore)

// --- Sync RBAC Data ---
if queryStore.IsEnabled() {
if err := wiring.SyncRBACFromStorage(context.Background(), blobStore, queryStore); err != nil {
log.Printf("Warning: Failed to sync RBAC data: %v", err)
}

// Sync existing units from storage to database
log.Println("Syncing existing units from storage to database...")
units, err := blobStore.List(context.Background(), "")
if err != nil {
log.Printf("Warning: Failed to list units from storage: %v", err)
} else {
for _, unit := range units {
// Always ensure unit exists first
if err := queryStore.SyncEnsureUnit(context.Background(), unit.ID); err != nil {
log.Printf("Warning: Failed to sync unit %s: %v", unit.ID, err)
continue
}

// Always sync metadata to update existing records
if err := queryStore.SyncUnitMetadata(context.Background(), unit.ID, unit.Size, unit.Updated); err != nil {
log.Printf("Warning: Failed to sync metadata for unit %s: %v", unit.ID, err)
}
}
log.Printf("Synced %d units from storage to database", len(units))
}
}

// --- Conditionally Apply Authorization Layer with a SMART CHECK ---
var finalStore storage.UnitStore

// Check if there are any RBAC roles defined in the database.
rbacIsConfigured, err := queryStore.HasRBACRoles(context.Background())
if err != nil {
log.Fatalf("Failed to check for RBAC configuration: %v", err)
}

// The condition is now two-part: Auth must be enabled AND RBAC roles must exist.
if !*authDisable && rbacIsConfigured {
log.Println("RBAC is ENABLED and CONFIGURED. Wrapping store with authorization layer.")
finalStore = storage.NewAuthorizingStore(orchestratingStore, queryStore)
} else {
if !*authDisable {
log.Println("RBAC is ENABLED but NOT CONFIGURED (no roles found). Authorization layer will be skipped.")
} else {
log.Println("RBAC is DISABLED via flag. Authorization layer will be skipped.")
}
finalStore = orchestratingStore
}

// Initialize analytics with system ID management (always create system ID)
analytics.InitGlobalWithSystemID("production", store)
analytics.InitGlobalWithSystemID("production", finalStore)

// Initialize system ID synchronously during startup
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
Expand All @@ -86,14 +171,21 @@ func main() {
e.HideBanner = true

// Middleware
e.Use(middleware.Logger())
e.Use(middleware.Recover())
e.Use(middleware.RequestID())
e.Use(middleware.Gzip())
e.Use(middleware.Secure())
e.Use(middleware.CORS())
e.Use(echomiddleware.Logger())
e.Use(echomiddleware.Recover())
e.Use(echomiddleware.RequestID())
e.Use(echomiddleware.Gzip())
e.Use(echomiddleware.Secure())
e.Use(echomiddleware.CORS())

api.RegisterRoutes(e, store, !*authDisable)

// Create a signer for JWTs (this may need to be configured from env vars)
signer, err := auth.NewSignerFromEnv()
if err != nil {
log.Fatalf("Failed to initialize JWT signer: %v", err)
}

api.RegisterRoutes(e, finalStore, !*authDisable, queryStore, blobStore, signer)

// Start server
go func() {
Expand Down Expand Up @@ -124,3 +216,4 @@ func main() {
analytics.SendEssential("server_shutdown_complete")
log.Println("Server shutdown complete")
}

21 changes: 20 additions & 1 deletion taco/cmd/taco/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,32 @@ module github.com/diggerhq/digger/opentaco/cmd/taco
go 1.24

require (
github.com/diggerhq/digger/opentaco/internal v0.0.0-00010101000000-000000000000
github.com/diggerhq/digger/opentaco/pkg/sdk v0.0.0
github.com/google/uuid v1.5.0
github.com/google/uuid v1.6.0
github.com/mr-tron/base58 v1.2.0
github.com/spf13/cobra v1.8.0
)

require (
github.com/aws/aws-sdk-go-v2 v1.38.1 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0 // indirect
github.com/aws/aws-sdk-go-v2/config v1.31.2 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.18.6 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.0 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.4 // indirect
github.com/aws/aws-sdk-go-v2/service/s3 v1.87.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.28.2 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.33.2 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.38.0 // indirect
github.com/aws/smithy-go v1.22.5 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
)
Expand Down
41 changes: 40 additions & 1 deletion taco/cmd/taco/go.sum
Original file line number Diff line number Diff line change
@@ -1,10 +1,49 @@
github.com/aws/aws-sdk-go-v2 v1.38.1 h1:j7sc33amE74Rz0M/PoCpsZQ6OunLqys/m5antM0J+Z8=
github.com/aws/aws-sdk-go-v2 v1.38.1/go.mod h1:9Q0OoGQoboYIAJyslFyF1f5K1Ryddop8gqMhWx/n4Wg=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0 h1:6GMWV6CNpA/6fbFHnoAjrv4+LGfyTqZz2LtCHnspgDg=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0/go.mod h1:/mXlTIVG9jbxkqDnr5UQNQxW1HRYxeGklkM9vAFeabg=
github.com/aws/aws-sdk-go-v2/config v1.31.2 h1:NOaSZpVGEH2Np/c1toSeW0jooNl+9ALmsUTZ8YvkJR0=
github.com/aws/aws-sdk-go-v2/config v1.31.2/go.mod h1:17ft42Yb2lF6OigqSYiDAiUcX4RIkEMY6XxEMJsrAes=
github.com/aws/aws-sdk-go-v2/credentials v1.18.6 h1:AmmvNEYrru7sYNJnp3pf57lGbiarX4T9qU/6AZ9SucU=
github.com/aws/aws-sdk-go-v2/credentials v1.18.6/go.mod h1:/jdQkh1iVPa01xndfECInp1v1Wnp70v3K4MvtlLGVEc=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.4 h1:lpdMwTzmuDLkgW7086jE94HweHCqG+uOJwHf3LZs7T0=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.4/go.mod h1:9xzb8/SV62W6gHQGC/8rrvgNXU6ZoYM3sAIJCIrXJxY=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.4 h1:IdCLsiiIj5YJ3AFevsewURCPV+YWUlOW8JiPhoAy8vg=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.4/go.mod h1:l4bdfCD7XyyZA9BolKBo1eLqgaJxl0/x91PL4Yqe0ao=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.4 h1:j7vjtr1YIssWQOMeOWRbh3z8g2oY/xPjnZH2gLY4sGw=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.4/go.mod h1:yDmJgqOiH4EA8Hndnv4KwAo8jCGTSnM5ASG1nBI+toA=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.4 h1:BE/MNQ86yzTINrfxPPFS86QCBNQeLKY2A0KhDh47+wI=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.4/go.mod h1:SPBBhkJxjcrzJBc+qY85e83MQ2q3qdra8fghhkkyrJg=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.0 h1:6+lZi2JeGKtCraAj1rpoZfKqnQ9SptseRZioejfUOLM=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.0/go.mod h1:eb3gfbVIxIoGgJsi9pGne19dhCBpK6opTYpQqAmdy44=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.4 h1:Beh9oVgtQnBgR4sKKzkUBRQpf1GnL4wt0l4s8h2VCJ0=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.4/go.mod h1:b17At0o8inygF+c6FOD3rNyYZufPw62o9XJbSfQPgbo=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.4 h1:ueB2Te0NacDMnaC+68za9jLwkjzxGWm0KB5HTUHjLTI=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.4/go.mod h1:nLEfLnVMmLvyIG58/6gsSA03F1voKGaCfHV7+lR8S7s=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.4 h1:HVSeukL40rHclNcUqVcBwE1YoZhOkoLeBfhUqR3tjIU=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.4/go.mod h1:DnbBOv4FlIXHj2/xmrUQYtawRFC9L9ZmQPz+DBc6X5I=
github.com/aws/aws-sdk-go-v2/service/s3 v1.87.1 h1:2n6Pd67eJwAb/5KCX62/8RTU0aFAAW7V5XIGSghiHrw=
github.com/aws/aws-sdk-go-v2/service/s3 v1.87.1/go.mod h1:w5PC+6GHLkvMJKasYGVloB3TduOtROEMqm15HSuIbw4=
github.com/aws/aws-sdk-go-v2/service/sso v1.28.2 h1:ve9dYBB8CfJGTFqcQ3ZLAAb/KXWgYlgu/2R2TZL2Ko0=
github.com/aws/aws-sdk-go-v2/service/sso v1.28.2/go.mod h1:n9bTZFZcBa9hGGqVz3i/a6+NG0zmZgtkB9qVVFDqPA8=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.33.2 h1:pd9G9HQaM6UZAZh19pYOkpKSQkyQQ9ftnl/LttQOcGI=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.33.2/go.mod h1:eknndR9rU8UpE/OmFpqU78V1EcXPKFTTm5l/buZYgvM=
github.com/aws/aws-sdk-go-v2/service/sts v1.38.0 h1:iV1Ko4Em/lkJIsoKyGfc0nQySi+v0Udxr6Igq+y9JZc=
github.com/aws/aws-sdk-go-v2/service/sts v1.38.0/go.mod h1:bEPcjW7IbolPfK67G1nilqWyoxYMSPrDiIQ3RdIdKgo=
github.com/aws/smithy-go v1.22.5 h1:P9ATCXPMb2mPjYBgueqJNCA5S9UfktsW0tTxi+a7eqw=
github.com/aws/smithy-go v1.22.5/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI=
github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o=
github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
Expand Down
Loading
Loading