From 89d95a2f85dc595369c4eaa393bd6e744ea42886 Mon Sep 17 00:00:00 2001 From: Dilip Sharma Date: Fri, 14 Aug 2020 19:15:27 +0530 Subject: [PATCH 1/2] Debug code. --- .gitignore | 3 + gcs/gcs.go | 188 ++++++++++++++++++++++++++++++++++++++++++++++++ gcs/gcs_test.go | 55 ++++++++++++++ gcs/sample.txt | 1 + 4 files changed, 247 insertions(+) create mode 100644 gcs/gcs.go create mode 100644 gcs/gcs_test.go create mode 100644 gcs/sample.txt diff --git a/.gitignore b/.gitignore index 52629c6..bbe5b5d 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,4 @@ test_env +.env +go.mod +go.sum \ No newline at end of file diff --git a/gcs/gcs.go b/gcs/gcs.go new file mode 100644 index 0000000..695f909 --- /dev/null +++ b/gcs/gcs.go @@ -0,0 +1,188 @@ +package gcs + +import ( + "context" + "fmt" + "io" + "io/ioutil" + "log" + "mime" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "regexp" + "strings" + "time" + + "cloud.google.com/go/storage" + "github.com/qor/oss" + "google.golang.org/api/iterator" +) + +// Client GCS storage +type Client struct { + *storage.Client + Config *Config +} + +// Config GCS client config +type Config struct { + Bucket string + Endpoint string +} + +// New initialize GCS storage +func New(config *Config) *Client { + + client := &Client{Config: config} + ctx := context.Background() + gcsclient, err := storage.NewClient(ctx) + if err != nil { + log.Println(err) + } + + client.Client = gcsclient + + return client +} + +// Get receive file with given path +func (client Client) Get(path string) (file *os.File, err error) { + readCloser, err := client.GetStream(path) + + ext := filepath.Ext(path) + pattern := fmt.Sprintf("gcs*%s", ext) + + if err == nil { + if file, err = ioutil.TempFile("/tmp", pattern); err == nil { + defer readCloser.Close() + _, err = io.Copy(file, readCloser) + file.Seek(0, 0) + } + } + + return file, err +} + +// GetStream get file as stream +func (client Client) GetStream(path string) (io.ReadCloser, error) { + log.Println("GetStream " + path) + bkt := client.Client.Bucket(client.Config.Bucket) + obj := bkt.Object(client.ToRelativePath(path)) + r, err := obj.NewReader(context.TODO()) + return r, err +} + +// Put store a reader into given path +func (client Client) Put(urlPath string, reader io.Reader) (*oss.Object, error) { + log.Println("Put " + urlPath) + if seeker, ok := reader.(io.ReadSeeker); ok { + seeker.Seek(0, 0) + } + + urlPath = client.ToRelativePath(urlPath) + buffer, err := ioutil.ReadAll(reader) + + fileType := mime.TypeByExtension(path.Ext(urlPath)) + if fileType == "" { + fileType = http.DetectContentType(buffer) + } + + bkt := client.Client.Bucket(client.Config.Bucket) + obj := bkt.Object(urlPath) + w := obj.NewWriter(context.TODO()) + w.ContentType = fileType + w.Write(buffer) + + if err := w.Close(); err != nil { + return nil, err + } + + now := time.Now() + return &oss.Object{ + Path: urlPath, + Name: filepath.Base(urlPath), + LastModified: &now, + StorageInterface: client, + }, err +} + +// Delete delete file +func (client Client) Delete(path string) error { + log.Println("Delete " + path) + bkt := client.Client.Bucket(client.Config.Bucket) + obj := bkt.Object(client.ToRelativePath(path)) + return obj.Delete(context.TODO()) +} + +// List list all objects under current path +func (client Client) List(path string) ([]*oss.Object, error) { + log.Println("List " + path) + var objects []*oss.Object + var prefix string + + if path != "" { + prefix = strings.Trim(path, "/") + "/" + } + + query := &storage.Query{Prefix: prefix} + bkt := client.Client.Bucket(client.Config.Bucket) + it := bkt.Objects(context.TODO(), query) + for { + attrs, err := it.Next() + if err == iterator.Done { + break + } + if err != nil { + log.Fatal(err) + } + + objects = append(objects, &oss.Object{ + Path: client.ToRelativePath(attrs.MediaLink), + Name: filepath.Base(attrs.Name), + LastModified: &attrs.Created, + StorageInterface: client, + }) + } + + return objects, nil +} + +// GetEndpoint get endpoint, FileSystem's endpoint is / +func (client Client) GetEndpoint() string { + endpoint := filepath.Join(client.Config.Endpoint, client.Config.Bucket) + return endpoint +} + +var urlRegexp = regexp.MustCompile("") + +// ToRelativePath process path to relative path +func (client Client) ToRelativePath(urlPath string) string { + log.Println("ToRelativePath " + urlPath) + if urlRegexp.MatchString(urlPath) { + if u, err := url.Parse(urlPath); err == nil { + return strings.TrimPrefix(u.Path, "/"+client.Config.Bucket) + } + } + + return "/" + strings.TrimPrefix(urlPath, "/") +} + +// GetURL get public accessible URL +func (client Client) GetURL(path string) (url string, err error) { + log.Println("GetURL " + path) + // if client.Endpoint == "" { + // if client.Config.ACL == s3.BucketCannedACLPrivate || client.Config.ACL == s3.BucketCannedACLAuthenticatedRead { + // getResponse, _ := client.S3.GetObjectRequest(&s3.GetObjectInput{ + // Bucket: aws.String(client.Config.Bucket), + // Key: aws.String(client.ToRelativePath(path)), + // }) + + // return getResponse.Presign(1 * time.Hour) + // } + // } + + return path, nil +} diff --git a/gcs/gcs_test.go b/gcs/gcs_test.go new file mode 100644 index 0000000..2affe9a --- /dev/null +++ b/gcs/gcs_test.go @@ -0,0 +1,55 @@ +package gcs_test + +import ( + "bufio" + "os" + "testing" + + "github.com/dilip640/oss/gcs" + "github.com/jinzhu/configor" +) + +type Config struct { + Bucket string `env:"QOR_GCS_BUCKET"` + Endpoint string `env:"QOR_GCS_ENDPOINT"` +} + +var ( + client *gcs.Client + config = Config{} +) + +func init() { + configor.Load(&config) + + client = gcs.New(&gcs.Config{Bucket: config.Bucket, Endpoint: config.Endpoint}) +} + +// func TestToRelativePath(t *testing.T) { +// urlMap := map[string]string{ +// "https://mybucket.s3.amazonaws.com/myobject.ext": "/myobject.ext", +// "https://qor-example.com/myobject.ext": "/myobject.ext", +// "//mybucket.s3.amazonaws.com/myobject.ext": "/myobject.ext", +// "http://mybucket.s3.amazonaws.com/myobject.ext": "/myobject.ext", +// "myobject.ext": "/myobject.ext", +// } + +// for url, path := range urlMap { +// if client.ToRelativePath(url) != path { +// t.Errorf("%v's relative path should be %v, but got %v", url, path, client.ToRelativePath(url)) +// } +// } +// } + +func TestUpload(t *testing.T) { + file, err := os.Open("sample.txt") + if err != nil { + t.Error(err) + return + } + + _, err = client.Put(file.Name(), bufio.NewReader(file)) + if err != nil { + t.Error(err) + } +} diff --git a/gcs/sample.txt b/gcs/sample.txt new file mode 100644 index 0000000..d64a3d9 --- /dev/null +++ b/gcs/sample.txt @@ -0,0 +1 @@ +sample From cae149e793a011f3a0f03cb8751f4dbec3f0d19e Mon Sep 17 00:00:00 2001 From: Dilip Sharma Date: Sat, 15 Aug 2020 01:31:27 +0530 Subject: [PATCH 2/2] Add implemet storage for gcs. --- gcs/README.md | 38 ++++++++++++++++++++++++++++++++++++++ gcs/gcs.go | 45 ++++++++++++++++++--------------------------- gcs/gcs_test.go | 47 +++++++++++++++++++++-------------------------- 3 files changed, 77 insertions(+), 53 deletions(-) create mode 100644 gcs/README.md diff --git a/gcs/README.md b/gcs/README.md new file mode 100644 index 0000000..3446b90 --- /dev/null +++ b/gcs/README.md @@ -0,0 +1,38 @@ +# Google Cloud Storage + +[GCS](https://pkg.go.dev/cloud.google.com/go/storage) backend for [QOR OSS](https://github.com/qor/oss) + +## Usage + +> Set ENV `GOOGLE_APPLICATION_CREDENTIALS` to service account + +```go +import "github.com/qor/oss/gcs" + +func main() { + storage := gcs.New(gcs.Config{ + Bucket: "bucket", + Endpoint: "https://storage.googleapis.com/", + }) + + // Save a reader interface into storage + storage.Put("/sample.txt", reader) + + // Get file with path + storage.Get("/sample.txt") + + // Get object as io.ReadCloser + storage.GetStream("/sample.txt") + + // Delete file with path + storage.Delete("/sample.txt") + + // List all objects under path + storage.List("/") + + // Get Public Accessible URL (useful if current file saved privately) + storage.GetURL("/sample.txt") +} +``` + + diff --git a/gcs/gcs.go b/gcs/gcs.go index 695f909..72862a0 100644 --- a/gcs/gcs.go +++ b/gcs/gcs.go @@ -68,16 +68,15 @@ func (client Client) Get(path string) (file *os.File, err error) { // GetStream get file as stream func (client Client) GetStream(path string) (io.ReadCloser, error) { - log.Println("GetStream " + path) bkt := client.Client.Bucket(client.Config.Bucket) obj := bkt.Object(client.ToRelativePath(path)) r, err := obj.NewReader(context.TODO()) + _, err = obj.Attrs(context.TODO()) return r, err } // Put store a reader into given path func (client Client) Put(urlPath string, reader io.Reader) (*oss.Object, error) { - log.Println("Put " + urlPath) if seeker, ok := reader.(io.ReadSeeker); ok { seeker.Seek(0, 0) } @@ -111,20 +110,20 @@ func (client Client) Put(urlPath string, reader io.Reader) (*oss.Object, error) // Delete delete file func (client Client) Delete(path string) error { - log.Println("Delete " + path) + path = strings.TrimPrefix(path, "/") bkt := client.Client.Bucket(client.Config.Bucket) obj := bkt.Object(client.ToRelativePath(path)) - return obj.Delete(context.TODO()) + err := obj.Delete(context.TODO()) + return err } // List list all objects under current path func (client Client) List(path string) ([]*oss.Object, error) { - log.Println("List " + path) var objects []*oss.Object var prefix string if path != "" { - prefix = strings.Trim(path, "/") + "/" + prefix = strings.Trim(path, "/") } query := &storage.Query{Prefix: prefix} @@ -140,7 +139,7 @@ func (client Client) List(path string) ([]*oss.Object, error) { } objects = append(objects, &oss.Object{ - Path: client.ToRelativePath(attrs.MediaLink), + Path: "/" + client.ToRelativePath(attrs.Name), Name: filepath.Base(attrs.Name), LastModified: &attrs.Created, StorageInterface: client, @@ -152,37 +151,29 @@ func (client Client) List(path string) ([]*oss.Object, error) { // GetEndpoint get endpoint, FileSystem's endpoint is / func (client Client) GetEndpoint() string { - endpoint := filepath.Join(client.Config.Endpoint, client.Config.Bucket) - return endpoint + u, err := url.Parse(client.Config.Endpoint) + if err != nil { + log.Println(err) + } + u.Path = path.Join(u.Path, client.Config.Bucket) + return u.String() } -var urlRegexp = regexp.MustCompile("") +var urlRegexp = regexp.MustCompile(`(https?:)?//((\w+).)+(\w+)/`) // ToRelativePath process path to relative path func (client Client) ToRelativePath(urlPath string) string { - log.Println("ToRelativePath " + urlPath) if urlRegexp.MatchString(urlPath) { if u, err := url.Parse(urlPath); err == nil { - return strings.TrimPrefix(u.Path, "/"+client.Config.Bucket) + urlPath = strings.TrimPrefix(u.Path, "/"+client.Config.Bucket+"/") + urlPath = strings.TrimPrefix(urlPath, "/") + return urlPath } } - - return "/" + strings.TrimPrefix(urlPath, "/") + return strings.TrimPrefix(urlPath, "/") } // GetURL get public accessible URL -func (client Client) GetURL(path string) (url string, err error) { - log.Println("GetURL " + path) - // if client.Endpoint == "" { - // if client.Config.ACL == s3.BucketCannedACLPrivate || client.Config.ACL == s3.BucketCannedACLAuthenticatedRead { - // getResponse, _ := client.S3.GetObjectRequest(&s3.GetObjectInput{ - // Bucket: aws.String(client.Config.Bucket), - // Key: aws.String(client.ToRelativePath(path)), - // }) - - // return getResponse.Presign(1 * time.Hour) - // } - // } - +func (client Client) GetURL(path string) (string, error) { return path, nil } diff --git a/gcs/gcs_test.go b/gcs/gcs_test.go index 2affe9a..7cf4699 100644 --- a/gcs/gcs_test.go +++ b/gcs/gcs_test.go @@ -1,11 +1,11 @@ package gcs_test import ( - "bufio" - "os" + "fmt" "testing" "github.com/dilip640/oss/gcs" + "github.com/dilip640/oss/tests" "github.com/jinzhu/configor" ) @@ -25,31 +25,26 @@ func init() { client = gcs.New(&gcs.Config{Bucket: config.Bucket, Endpoint: config.Endpoint}) } -// func TestToRelativePath(t *testing.T) { -// urlMap := map[string]string{ -// "https://mybucket.s3.amazonaws.com/myobject.ext": "/myobject.ext", -// "https://qor-example.com/myobject.ext": "/myobject.ext", -// "//mybucket.s3.amazonaws.com/myobject.ext": "/myobject.ext", -// "http://mybucket.s3.amazonaws.com/myobject.ext": "/myobject.ext", -// "myobject.ext": "/myobject.ext", -// } - -// for url, path := range urlMap { -// if client.ToRelativePath(url) != path { -// t.Errorf("%v's relative path should be %v, but got %v", url, path, client.ToRelativePath(url)) -// } -// } -// } - -func TestUpload(t *testing.T) { - file, err := os.Open("sample.txt") - if err != nil { - t.Error(err) - return +func TestAll(t *testing.T) { + fmt.Println("testing GCS with public ACL") + tests.TestAll(client, t) + + fmt.Println("testing GCS with private ACL") + privateClient := gcs.New(&gcs.Config{Bucket: config.Bucket, Endpoint: config.Endpoint}) + tests.TestAll(privateClient, t) +} + +func TestToRelativePath(t *testing.T) { + urlMap := map[string]string{ + "https://storage.googleapis.com/pelto-test/myobject.ext": "myobject.ext", + "//storage.googleapis.com/pelto-test/myobject.ext": "myobject.ext", + "gs://pelt-test/myobject.ext": "myobject.ext", + "myobject.ext": "myobject.ext", } - _, err = client.Put(file.Name(), bufio.NewReader(file)) - if err != nil { - t.Error(err) + for url, path := range urlMap { + if client.ToRelativePath(url) != path { + t.Errorf("%v's relative path should be %v, but got %v", url, path, client.ToRelativePath(url)) + } } }