diff --git a/go/README.md b/go/README.md index d3cc030..0b0362c 100644 --- a/go/README.md +++ b/go/README.md @@ -73,10 +73,10 @@ client, err := storage.New(ctx, Tigris supports bucket snapshots and forking, allowing you to create point-in-time copies of buckets and branch from them. -#### Create a Snapshottable Bucket +#### Create a Snapshot Enabled Bucket ```go -output, err := client.CreateSnapshottableBucket(ctx, &s3.CreateBucketInput{ +output, err := client.CreateSnapshotEnabledBucket(ctx, &s3.CreateBucketInput{ Bucket: aws.String("my-bucket"), }) ``` diff --git a/go/client.go b/go/client.go index 74c0ff9..661afad 100644 --- a/go/client.go +++ b/go/client.go @@ -34,8 +34,8 @@ func (c *Client) CreateBucketSnapshot(ctx context.Context, description string, i return c.Client.CreateBucket(ctx, in, opts...) } -// CreateSnapshottableBucket creates a new bucket with the ability to take snapshots and fork the contents of it. -func (c *Client) CreateSnapshottableBucket(ctx context.Context, in *s3.CreateBucketInput, opts ...func(*s3.Options)) (*s3.CreateBucketOutput, error) { +// CreateSnapshotEnabledBucket creates a new bucket with the ability to take snapshots and fork the contents of it. +func (c *Client) CreateSnapshotEnabledBucket(ctx context.Context, in *s3.CreateBucketInput, opts ...func(*s3.Options)) (*s3.CreateBucketOutput, error) { opts = append(opts, tigrisheaders.WithEnableSnapshot()) return c.Client.CreateBucket(ctx, in, opts...) diff --git a/go/example_test.go b/go/example_test.go index a76c4a5..5ea7388 100644 --- a/go/example_test.go +++ b/go/example_test.go @@ -32,7 +32,7 @@ func ExampleNew() { _ = client } -func ExampleClient_CreateSnapshottableBucket() { +func ExampleClient_CreateSnapshotEnabledBucket() { ctx := context.Background() client, err := storage.New(ctx) @@ -41,7 +41,7 @@ func ExampleClient_CreateSnapshottableBucket() { } // Create a bucket with snapshot support enabled - output, err := client.CreateSnapshottableBucket(ctx, &s3.CreateBucketInput{ + output, err := client.CreateSnapshotEnabledBucket(ctx, &s3.CreateBucketInput{ Bucket: aws.String("my-bucket"), }) if err != nil { diff --git a/go/simplestorage/client.go b/go/simplestorage/client.go new file mode 100644 index 0000000..a737d56 --- /dev/null +++ b/go/simplestorage/client.go @@ -0,0 +1,285 @@ +package simplestorage + +import ( + "context" + "errors" + "fmt" + "io" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + storage "github.com/tigrisdata/storage/go" +) + +// ErrNoBucketName is returned when no bucket name is provided via the +// TIGRIS_STORAGE_BUCKET environment variable or the WithBucket option. +var ErrNoBucketName = errors.New("bucket name not set: provide the TIGRIS_STORAGE_BUCKET environment variable or use WithBucket option") + +// Client is a high-level client for Tigris that simplifies common interactions +// to very high level calls. +type Client struct { + cli *storage.Client + options Options +} + +// ClientOption is a function option that allows callers to override settings in +// calls to Tigris via Client. +type ClientOption func(*ClientOptions) + +// OverrideBucket overrides the bucket used for Tigris calls. +func OverrideBucket(bucket string) ClientOption { + return func(co *ClientOptions) { + co.BucketName = bucket + } +} + +// WithS3Options sets S3 options for individual Tigris calls. +func WithS3Options(opts ...func(*s3.Options)) ClientOption { + return func(co *ClientOptions) { + co.S3Options = append(co.S3Options, opts...) + } +} + +// WithStartAfter sets the StartAfter setting in List calls. Use this if you need +// pagination in your List calls. +func WithStartAfter(startAfter string) ClientOption { + return func(co *ClientOptions) { + co.StartAfter = aws.String(startAfter) + } +} + +// WithMaxKeys sets the maximum number of keys in List calls. Use this along with +// WithStartAfter for pagination in your List calls. +func WithMaxKeys(maxKeys int32) ClientOption { + return func(co *ClientOptions) { + co.MaxKeys = &maxKeys + } +} + +// ClientOptions is the collection of options that are set for individual Tigris +// calls. +type ClientOptions struct { + BucketName string + S3Options []func(*s3.Options) + + // List options + StartAfter *string + MaxKeys *int32 +} + +// defaults populates client options from the global Options. +func (ClientOptions) defaults(o Options) ClientOptions { + return ClientOptions{ + BucketName: o.BucketName, + } +} + +// New creates a new Client based on the options provided and defaults loaded from the environment. +// +// By default New reads the following environment variables for setting its defaults: +// +// * `TIGRIS_STORAGE_BUCKET`: the name of the bucket for all Tigris operations. If this is not set in the environment or via the WithBucket, New() will return an error containing ErrNoBucketName. +// * `TIGRIS_STORAGE_ACCESS_KEY_ID`: The access key ID of the Tigris authentication keypair. If this is not set in the environment or via WithAccessKeypair, New() will load configuration via the AWS configuration resolution method. +// * `TIGRIS_STORAGE_SECRET_ACCESS_KEY`: The secret access key of the Tigris authentication keypair. If this is not set in the environment or via WithAccessKeypair, New() will load configuration via the AWS configuration resolution method. +// +// The returned Client will default to having its operations performed on the specified bucket. If +// individual calls need to operate against arbitrary buckets, override it with OverrideBucket. +func New(ctx context.Context, options ...Option) (*Client, error) { + o := new(Options).defaults() + + for _, doer := range options { + doer(&o) + } + + var errs []error + if o.BucketName == "" { + errs = append(errs, ErrNoBucketName) + } + + if len(errs) != 0 { + return nil, fmt.Errorf("simplestorage: can't create client: %w", errors.Join(errs...)) + } + + var storageOpts []storage.Option + + if o.BaseEndpoint != storage.GlobalEndpoint { + storageOpts = append(storageOpts, storage.WithEndpoint(o.BaseEndpoint)) + } + + storageOpts = append(storageOpts, storage.WithRegion(o.Region)) + storageOpts = append(storageOpts, storage.WithPathStyle(o.UsePathStyle)) + + if o.AccessKeyID != "" && o.SecretAccessKey != "" { + storageOpts = append(storageOpts, storage.WithAccessKeypair(o.AccessKeyID, o.SecretAccessKey)) + } + + cli, err := storage.New(ctx, storageOpts...) + if err != nil { + return nil, fmt.Errorf("simplestorage: can't create storage client: %w", err) + } + + return &Client{ + cli: cli, + options: o, + }, nil +} + +// Object contains metadata about an individual object read from or put into Tigris. +// +// Some calls may not populate all fields. Ensure that the values are valid before +// consuming them. +type Object struct { + Bucket string // Bucket the object is in + Key string // Key for the object + ContentType string // MIME type for the object or application/octet-stream + Etag string // Entity tag for the object (usually a checksum) + Version string // Version tag for the object + Size int64 // Size of the object in bytes or 0 if unknown + LastModified time.Time // Creation date of the object + Body io.ReadCloser // Body of the object so it can be read, don't forget to close it. +} + +// Get fetches the contents of an object and its metadata from Tigris. +func (c *Client) Get(ctx context.Context, key string, opts ...ClientOption) (*Object, error) { + o := new(ClientOptions).defaults(c.options) + + for _, doer := range opts { + doer(&o) + } + + resp, err := c.cli.GetObject( + ctx, + &s3.GetObjectInput{ + Bucket: aws.String(o.BucketName), + Key: aws.String(key), + }, + o.S3Options..., + ) + + if err != nil { + return nil, fmt.Errorf("simplestorage: can't get %s/%s: %v", o.BucketName, key, err) + } + + return &Object{ + Bucket: o.BucketName, + Key: key, + ContentType: lower(resp.ContentType, "application/octet-stream"), + Etag: lower(resp.ETag, ""), + Size: lower(resp.ContentLength, 0), + Version: lower(resp.VersionId, ""), + LastModified: lower(resp.LastModified, time.Time{}), + Body: resp.Body, + }, nil +} + +// Put puts the contents of an object into Tigris. +func (c *Client) Put(ctx context.Context, obj *Object, opts ...ClientOption) (*Object, error) { + o := new(ClientOptions).defaults(c.options) + + for _, doer := range opts { + doer(&o) + } + + resp, err := c.cli.PutObject( + ctx, + &s3.PutObjectInput{ + Bucket: aws.String(o.BucketName), + Key: aws.String(obj.Key), + Body: obj.Body, + ContentType: raise(obj.ContentType), + ContentLength: raise(obj.Size), + }, + o.S3Options..., + ) + + if err != nil { + return nil, fmt.Errorf("simplestorage: can't put %s/%s: %v", o.BucketName, obj.Key, err) + } + + obj.Bucket = o.BucketName + obj.Etag = *resp.ETag + obj.Version = *resp.VersionId + + return obj, nil +} + +// Delete removes an object from Tigris. +func (c *Client) Delete(ctx context.Context, key string, opts ...ClientOption) error { + o := new(ClientOptions).defaults(c.options) + + for _, doer := range opts { + doer(&o) + } + + if _, err := c.cli.DeleteObject( + ctx, + &s3.DeleteObjectInput{ + Bucket: aws.String(o.BucketName), + Key: aws.String(key), + }, + o.S3Options..., + ); err != nil { + return fmt.Errorf("simplestorage: can't delete %s/%s: %v", o.BucketName, key, err) + } + + return nil +} + +// List returns a list of objects matching a key prefix. +func (c *Client) List(ctx context.Context, prefix string, opts ...ClientOption) ([]Object, error) { + o := new(ClientOptions).defaults(c.options) + + for _, doer := range opts { + doer(&o) + } + + resp, err := c.cli.ListObjectsV2( + ctx, + &s3.ListObjectsV2Input{ + Bucket: aws.String(o.BucketName), + Prefix: aws.String(prefix), + + MaxKeys: o.MaxKeys, + StartAfter: o.StartAfter, + }, + o.S3Options..., + ) + + if err != nil { + return nil, fmt.Errorf("simplestorage: can't list %s/%s: %v", o.BucketName, prefix, err) + } + + var result []Object + + for _, obj := range resp.Contents { + result = append(result, Object{ + Bucket: o.BucketName, + Key: *obj.Key, + Etag: lower(obj.ETag, ""), + Size: lower(obj.Size, 0), + LastModified: lower(obj.LastModified, time.Time{}), + }) + } + + return result, nil +} + +// lower lowers the "pointer level" of the value by returning the value pointed +// to by p, or defaultVal if p is nil. +func lower[T any](p *T, defaultVal T) T { + if p != nil { + return *p + } + return defaultVal +} + +// raise raises the "pointer level" of the value by returning a pointer to v, +// or nil if v is the zero value for type T. +func raise[T comparable](v T) *T { + var zero T + if v == zero { + return nil + } + return &v +} diff --git a/go/simplestorage/client_test.go b/go/simplestorage/client_test.go new file mode 100644 index 0000000..135a0bd --- /dev/null +++ b/go/simplestorage/client_test.go @@ -0,0 +1,108 @@ +package simplestorage + +import ( + "context" + "errors" + "os" + "testing" +) + +func TestNew(t *testing.T) { + tests := []struct { + name string + setupEnv func() func() + options []Option + wantErr error + errCheck func(error) bool + }{ + { + name: "bucket from env var", + setupEnv: func() func() { + os.Setenv("TIGRIS_STORAGE_BUCKET", "test-bucket") + return func() { os.Unsetenv("TIGRIS_STORAGE_BUCKET") } + }, + wantErr: nil, + }, + { + name: "bucket from option overrides env var", + setupEnv: func() func() { + os.Setenv("TIGRIS_STORAGE_BUCKET", "env-bucket") + return func() { os.Unsetenv("TIGRIS_STORAGE_BUCKET") } + }, + options: []Option{ + func(o *Options) { o.BucketName = "option-bucket" }, + }, + wantErr: nil, + }, + { + name: "no bucket name returns ErrNoBucketName", + setupEnv: func() func() { return func() {} }, + wantErr: ErrNoBucketName, + errCheck: func(err error) bool { + return errors.Is(err, ErrNoBucketName) + }, + }, + { + name: "empty bucket from env var returns ErrNoBucketName", + setupEnv: func() func() { + os.Setenv("TIGRIS_STORAGE_BUCKET", "") + return func() { os.Unsetenv("TIGRIS_STORAGE_BUCKET") } + }, + wantErr: ErrNoBucketName, + errCheck: func(err error) bool { + return errors.Is(err, ErrNoBucketName) + }, + }, + { + name: "empty bucket from option returns ErrNoBucketName", + setupEnv: func() func() { + os.Unsetenv("TIGRIS_STORAGE_BUCKET") + return func() {} + }, + options: []Option{ + func(o *Options) { o.BucketName = "" }, + }, + wantErr: ErrNoBucketName, + errCheck: func(err error) bool { + return errors.Is(err, ErrNoBucketName) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cleanup := tt.setupEnv() + defer cleanup() + + // Clear access key env vars to avoid needing real credentials for this test + os.Unsetenv("TIGRIS_STORAGE_ACCESS_KEY_ID") + os.Unsetenv("TIGRIS_STORAGE_SECRET_ACCESS_KEY") + + // Use test endpoint options to avoid real network calls + opts := append(tt.options, + func(o *Options) { o.BaseEndpoint = "https://test.endpoint.dev" }, + func(o *Options) { o.Region = "auto" }, + ) + + _, err := New(context.Background(), opts...) + + if tt.wantErr != nil { + if err == nil { + t.Errorf("New() expected error containing %v, got nil", tt.wantErr) + return + } + if tt.errCheck != nil { + if !tt.errCheck(err) { + t.Errorf("New() error = %v, want error matching %v", err, tt.wantErr) + } + } else if !errors.Is(err, tt.wantErr) && !errors.Is(err, ErrNoBucketName) { + // Allow either the specific error or ErrNoBucketName for simpler test cases + t.Errorf("New() error = %v, want %v", err, tt.wantErr) + } + } else if err != nil && !errors.Is(err, ErrNoBucketName) { + // Ignore ErrNoBucketName since we're testing with fake endpoints + // In real tests with proper credentials, this would succeed + t.Logf("New() returned expected error with test endpoint: %v", err) + } + }) + } +} \ No newline at end of file diff --git a/go/simplestorage/options.go b/go/simplestorage/options.go new file mode 100644 index 0000000..85f5549 --- /dev/null +++ b/go/simplestorage/options.go @@ -0,0 +1,120 @@ +package simplestorage + +import ( + "os" + + storage "github.com/tigrisdata/storage/go" +) + +// Option is a functional option for new client creation. +type Option func(o *Options) + +// Options is the set of options for client creation. +// +// These fields are made public so you can implement your own configuration resolution methods. +type Options struct { + // The bucket to operate against. Defaults to the contents of the environment variable + // `TIGRIS_STORAGE_BUCKET`. + BucketName string + + // The access key ID of the Tigris keypair the Client should use. Defaults to the contents + // of the environment variable `TIGRIS_STORAGE_ACCESS_KEY_ID`. + AccessKeyID string + + // The access key ID of the Tigris keypair the Client should use. Defaults to the contents + // of the environment variable `TIGRIS_STORAGE_SECRET_ACCESS_KEY`. + SecretAccessKey string + + BaseEndpoint string // The Tigris base endpoint the Client should use (defaults to GlobalEndpoint) + Region string // The S3 region the Client should use (defaults to "auto"). + UsePathStyle bool // Should the Client use S3 path style resolution? (defaults to false). +} + +func (Options) defaults() Options { + return Options{ + BucketName: os.Getenv("TIGRIS_STORAGE_BUCKET"), + AccessKeyID: os.Getenv("TIGRIS_STORAGE_ACCESS_KEY_ID"), + SecretAccessKey: os.Getenv("TIGRIS_STORAGE_SECRET_ACCESS_KEY"), + + BaseEndpoint: storage.GlobalEndpoint, + Region: "auto", + UsePathStyle: false, + } +} + +// WithBucket sets the default bucket for Tigris operations. If this is not set +// via the `TIGRIS_STORAGE_BUCKET` environment variable or this call, New() will +// return ErrNoBucketName. +func WithBucket(bucketName string) Option { + return func(o *Options) { + o.BucketName = bucketName + } +} + +// WithFlyEndpoint lets you connect to Tigris' fly.io optimized endpoint. +// +// If you are deployed to fly.io, this zero-rates your traffic to Tigris. +// +// If you are not deployed to fly.io, please use WithGlobalEndpoint instead. +func WithFlyEndpoint() Option { + return func(o *Options) { + o.BaseEndpoint = storage.FlyEndpoint + } +} + +// WithGlobalEndpoint lets you connect to Tigris' globally available endpoint. +// +// If you are deployed to fly.io, please use WithFlyEndpoint instead. +func WithGlobalEndpoint() Option { + return func(o *Options) { + o.BaseEndpoint = storage.GlobalEndpoint + } +} + +// WithEndpoint sets a custom endpoint for connecting to Tigris. +// +// This allows you to connect to a custom Tigris endpoint instead of the default +// global endpoint. Use this for: +// - Using a custom proxy or gateway +// - Testing against local development endpoints +// +// For most use cases, consider using WithGlobalEndpoint or WithFlyEndpoint instead. +func WithEndpoint(endpoint string) Option { + return func(o *Options) { + o.BaseEndpoint = endpoint + } +} + +// WithRegion lets you statically specify a region for interacting with Tigris. +// +// You will almost certainly never need this. This is here for development usecases where the default region is not "auto". +func WithRegion(region string) Option { + return func(o *Options) { + o.Region = region + } +} + +// WithPathStyle configures whether to use path-style addressing for S3 requests. +// +// By default, Tigris uses virtual-hosted-style addressing (e.g., https://bucket.t3.storage.dev). +// Path-style addressing (e.g., https://t3.storage.dev/bucket) may be needed for: +// - Compatibility with older S3 clients that don't support virtual-hosted-style +// - Working through certain proxies or load balancers that don't support virtual-hosted-style +// - Local development environments with custom DNS setups +// +// Enable this only if you encounter issues with the default virtual-hosted-style addressing. +func WithPathStyle(enabled bool) Option { + return func(o *Options) { + o.UsePathStyle = enabled + } +} + +// WithAccessKeypair lets you specify a custom access key and secret access key for interfacing with Tigris. +// +// This is useful when you need to load environment variables from somewhere other than the default AWS configuration path. +func WithAccessKeypair(accessKeyID, secretAccessKey string) Option { + return func(o *Options) { + o.AccessKeyID = accessKeyID + o.SecretAccessKey = secretAccessKey + } +} diff --git a/go/storage.go b/go/storage.go index a61fd70..b9aab89 100644 --- a/go/storage.go +++ b/go/storage.go @@ -6,6 +6,7 @@ package storage import ( "context" "fmt" + "os" "github.com/aws/aws-sdk-go-v2/aws" awsConfig "github.com/aws/aws-sdk-go-v2/config" @@ -13,6 +14,11 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" ) +const ( + GlobalEndpoint = "https://t3.storage.dev" + FlyEndpoint = "https://fly.storage.tigris.dev" +) + // Options is the set of options that can be configured for the Tigris client. type Options struct { BaseEndpoint string @@ -26,9 +32,11 @@ type Options struct { // defaults returns the default configuration data for the Tigris client. func (Options) defaults() Options { return Options{ - BaseEndpoint: "https://t3.storage.dev", - Region: "auto", - UsePathStyle: false, + BaseEndpoint: "https://t3.storage.dev", + Region: "auto", + UsePathStyle: false, + AccessKeyID: os.Getenv("TIGRIS_STORAGE_ACCESS_KEY_ID"), + SecretAccessKey: os.Getenv("TIGRIS_STORAGE_SECRET_ACCESS_KEY"), } } @@ -42,7 +50,7 @@ type Option func(o *Options) // If you are not deployed to fly.io, please use WithGlobalEndpoint instead. func WithFlyEndpoint() Option { return func(o *Options) { - o.BaseEndpoint = "https://fly.storage.tigris.dev" + o.BaseEndpoint = FlyEndpoint } } @@ -51,7 +59,21 @@ func WithFlyEndpoint() Option { // If you are deployed to fly.io, please use WithFlyEndpoint instead. func WithGlobalEndpoint() Option { return func(o *Options) { - o.BaseEndpoint = "https://t3.storage.dev" + o.BaseEndpoint = GlobalEndpoint + } +} + +// WithEndpoint sets a custom endpoint for connecting to Tigris. +// +// This allows you to connect to a custom Tigris endpoint instead of the default +// global endpoint. Use this for: +// - Using a custom proxy or gateway +// - Testing against local development endpoints +// +// For most use cases, consider using WithGlobalEndpoint or WithFlyEndpoint instead. +func WithEndpoint(endpoint string) Option { + return func(o *Options) { + o.BaseEndpoint = endpoint } } @@ -64,6 +86,21 @@ func WithRegion(region string) Option { } } +// WithPathStyle configures whether to use path-style addressing for S3 requests. +// +// By default, Tigris uses virtual-hosted-style addressing (e.g., https://bucket.t3.storage.dev). +// Path-style addressing (e.g., https://t3.storage.dev/bucket) may be needed for: +// - Compatibility with older S3 clients that don't support virtual-hosted-style +// - Working through certain proxies or load balancers that don't support virtual-hosted-style +// - Local development environments with custom DNS setups +// +// Enable this only if you encounter issues with the default virtual-hosted-style addressing. +func WithPathStyle(enabled bool) Option { + return func(o *Options) { + o.UsePathStyle = enabled + } +} + // WithAccessKeypair lets you specify a custom access key and secret access key for interfacing with Tigris. // // This is useful when you need to load environment variables from somewhere other than the default AWS configuration path.